diff --git a/.gitignore b/.gitignore index d3cc6f85..62381cdf 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,13 @@ TeamProjectOutputEncrypted.csv /src/it/resources/com/phasmidsoftware/examples/crime/2023-01-metropolitan-street.csv hprof.samples.txt + +crimeSample.csv + +src/junk.csv + +tmp/Crime.use.Resource.csv + +tmp/other-render to CSV.csv + +tmp/Table-write Table To File.csv diff --git a/LICENSE b/LICENSE index c5864716..a715a87f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,204 +1,4 @@ - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] + Copyright 2023 Phasmid Software Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -210,4 +10,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. + limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md index 16e04b67..e2953635 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ A functional parser of tables implemented in Scala. Typically, the input is in the form of a "CSV" (comma-separated-values) file. However, it is perfectly possible to parse other formats. -_TableParser_ aims to make it as simple as possible to ingest a fully-typed tabular dataset. +_TableParser_ aims to make it as simple as possible to ingest a strictly-typed tabular dataset. The principal mechanism for this is the use of case classes to specify the types of fields in the dataset. All conversions from strings to standard types are performed automatically. For non-standard types, it suffices simply to provide an implicit converter of the form _String=>T_. @@ -34,10 +34,30 @@ together with something like, for instance, a Json writer. Quick Intro =========== -The simplest way to get an introduction to TableParser is to consult the airbnb.sc and movie.sc worksheets. +The simplest way to get an introduction to _TableParser_ is to consult the airbnb.sc and movie.sc worksheets. These give detailed descriptions of each stage of the process. -Another way to see how it works is to look at this application Pairings which takes a CSV file, parses it, transforms the data, +Take a look also at the _Main_ object in the _Crime.scala_ module (it's under the _test_ directory). +The model is relatively simple, but not too simple. +There are 12 columns in total, but five have been grouped into _CrimeLocation_, with the remaining seven at the top level, i.e., in _Crime_. +The members _CrimeID_ and _CrimeLocation_ are optional. +A sample data file is located in com/phasmidsoftware/examples/crime/2023-01-metropolitan-street-sample.csv. +The full file can be downloaded from Kaggle (see code) in _Crime_. + +One possibility is to run an analysis on the data (see CrimeSpec: Crime/be ingested and analyzed as a RawTable). +In order to read the dataset as a _Table\[Crime]_. + + import CrimeParser._ + val cti: IO[Table[Crime]] = Table.parseResource(Crime.sampleFile, classOf[Crime]) + matchIO(cti, Timeout(Span(60, Seconds))) { + case table@HeadedTable(_, _) => + // operate on table + } + +Notice that the parsed table is wrapped inside _IO_, the Cats I/O monad. +This has some technical advantages over using _Future_ or _Try_, which we won't detail here. + +Another way to see how it works is to look at this application _Pairings_ which takes a CSV file, parses it, transforms the data, and outputs a JSON file. This way of parsing is a little different from what is shown in the worksheets. But both are effective. @@ -49,7 +69,7 @@ The minimum code necessary to read parse the CSV file as a table of "Player"s, u def cellParser: CellParser[Player] = cellParser2(apply) } - val pty: Try[Table[Player]] = Table.parseFile[Table[Player]]("players.csv") + val pty: IO[Table[Player]] = Table.parseFile[Table[Player]]("players.csv") This assumes that the source input file ("players.csv") contains a header row which includes column names corresponding to the parameters of the case class _Player_ (in this case "first" and "last"). @@ -90,14 +110,24 @@ you will start with (1) and run an analysis on the columns to help you design th For the first option, you will do something like the following (see the _AnalysisSpec_ unit tests): - Table.parseResourceRaw(resourceName) match { - case Success(t@HeadedTable(_, _)) => println(Analysis(t)) - case _ => + private val sampleFile = "2023-01-metropolitan-street-sample.csv" + private val triedSampleResource: Try[URL] = FP.resource[Analysis](sampleFile) + val fraction = 4 + val parser = RawTableParser().setPredicate(TableParser.sampler(fraction)) + val ui = IOUsing(for (u <- triedSampleResource) yield Source.fromURL(u)) { + s => parser.doParse(s) map (rawTable => println(Analysis(rawTable))) } + ui.unsafeRunSync() + +This analysis will give you a list of columns, each showing its name, size, and +whether it is optional (i.e. contains nulls), together with an _Analytic_: +* if it's a numerical column: its range, mean, and standard deviation. +* if it's a column made up of a relatively small number of classes: +a histogram giving the class names with frequency, in order of decreasing frequency. -This analysis will give you a list of columns, each showing its name, -whether it is optional (i.e. contains nulls), and (if it's a numerical column), -its range, mean, and standard deviation. +Note the use of the predicate and sampler. +This allows you to randomly choose a subset of the rows. +In the example given, approximately one quarter of the rows will be chosen. Incidentally, this raw parser has three signatures, one for resources, one for files, and one for a sequence of Strings. And the default for raw row parsing is to allow quoted strings to span multiple lines. @@ -140,44 +170,61 @@ See section on _CellParsers_ below. ## Table -The _Table_ class, which implements _Iterable\[Row]_, also has several methods for manipulation: +The _Table_ class, which extends _Iterable\[Row]_, also has several methods for manipulation: ### query methods -* def rows: Seq\[Row] +* def content: Content\[Row] * def maybeHeader: Option\[Header] * def toCSV(implicit renderer: CsvRenderer\[Row], generator: CsvProductGenerator\[Row], csvAttributes: CsvAttributes): Iterable\[String] * def maybeColumnNames: Option\[Seq\[String]] * def column(name: String): Iterator\[Option\[String]] +* writeCSVFile(file: File)(implicit renderer: CsvRenderer\[Row], generator: CsvGenerator\[Row], ordering: Ordering\[Row], csvAttributes: CsvAttributes): Unit +* def writeCSVFileEncrypted[A: HexEncryption](file: File)(implicit renderer: CsvRenderer\[Row], generator: CsvGenerator\[Row], ordering: Ordering\[Row], hasKey: HasKey\[Row], csvAttributes: CsvAttributes): Unit ### transformation methods +* def filter(p: Row => Boolean): Table\[Row] +* def filterNot(p: Row => Boolean): Table\[Row] +* def filterValid(implicit rv: Validity\[Row]): Table\[Row] +* def map\[S](f: Row => S): Table\[S] * def flatMap\[U](f: Row => Iterable\[U]): Table\[U] +* def mapOptional\[S](f: Row => Option\[S]): Table\[S] +* def unit\[S](sc: Content\[S], maybeHeader: Option\[Header]): Table\[S] * def unit\[S](rows: Iterable\[S], maybeHeader: Option\[Header]): Table\[S] * def ++\[U >: Row](table: Table\[U]): Table\[U] +* def zip\[R](rt: Table\[R]): Table\[(Row, R)] * def processRows\[S](f: Iterable\[Row] => Iterable\[S]): Table\[S] * def processRows\[R, S](f: (Iterable\[Row], Iterable\[R]) => Iterable\[S])(other: Table\[R]): Table\[S] * def sort\[S >: Row : Ordering]: Table\[S] * def select(range: Range): Table\[Row] * def select(n: Int): Table\[Row] +* def drop(n: Int): Table\[Row] +* def dropWhile(p: Row => Boolean): Table\[Row] +* def take(n: Int): Table\[Row] +* def takeWhile(p: Row => Boolean): Table\[Row] +* def sample(n: Int)(implicit random: Random): Table\[Row] +* def slice(from: Int, until: Int): Table\[Row] * lazy val shuffle: Table\[Row] - It is to be expected that _join_ methods will be added later (based upon the second signature of processRows). -The following **object** methods are available for parsing text: -* def parse\[T: TableParser](ws: Seq\[String]): Try\[T] -* def parse\[T: TableParser](ws: Iterator\[String]): Try\[T] -* def parse\[T: TableParser](x: => Source): Try\[T] -* def parse\[T: TableParser](u: URI)(implicit codec: Codec): Try\[T] -* def parse\[T: TableParser](u: URI, enc: String): Try\[T] -* def parseInputStream\[T: TableParser](i: InputStream)(implicit codec: Codec): Try\[T] -* def parseInputStream\[T: TableParser](i: InputStream, enc: String): Try\[T] -* def parseFile\[T: TableParser](f: File)(implicit codec: Codec): Try\[T] -* def parseFile\[T: TableParser](f: File, enc: String): Try\[T] -* def parseFile\[T: TableParser](pathname: String)(implicit codec: Codec): Try\[T] -* def parseFile\[T: TableParser](pathname: String, enc: String): Try\[T] -* def parseResource\[T: TableParser](s: String, clazz: Class\[_] = getClass)(implicit codec: Codec): Try\[T] -* def parseResource\[T: TableParser](u: URL, enc: String): Try\[T] -* def parseResource\[T: TableParser](u: URL)(implicit codec: Codec): Try\[T] -* def parseSequence\[T: TableParser](wss: Seq\[Seq\[String]]): Try\[T] +The following **object** methods are available for parsing text in _Table_: +* def parse\[T: TableParser](ws: Iterable\[String]): IO\[T] +* def parse\[T: TableParser](ws: Iterator\[String]): IO\[T] +* def parseSource\[T: TableParser](x: => Source): IO\[T] +* def parse\[T: TableParser](si: => IO\[Source]): IO\[T] +* def parse\[T: TableParser](u: URI)(implicit codec: Codec): IO\[T] +* def parse\[T: TableParser](u: URI, enc: String): IO\[T] +* def parseInputStream\[T: TableParser](i: InputStream)(implicit codec: Codec): IO\[T] +* def parseInputStream\[T: TableParser](i: InputStream, enc: String): IO\[T] +* def parseFile\[T: TableParser](f: File)(implicit codec: Codec): IO\[T] +* def parseFile\[T: TableParser](f: File, enc: String): IO\[T] +* def parseFile\[T: TableParser](pathname: String)(implicit codec: Codec): IO\[T] +* def parseFile\[T: TableParser](pathname: String, enc: String): IO\[T] +* def parseResource\[T: TableParser](s: String, clazz: Class\[_] = getClass)(implicit codec: Codec): IO\[T] +* def parseResource\[T: TableParser](u: URL, enc: String): IO\[T] +* def parseResource\[T: TableParser](u: URL)(implicit codec: Codec): IO\[T] +* def parseSequence\[T: TableParser](wss: Seq\[Seq\[String]]): IO\[T] +* def parseFileRaw(f: File, predicate: Try\[RawRow] => Boolean, maybeFixedHeader: Option\[Header] = None, forgiving: Boolean = true)(implicit codec: Codec): IO\[Table\[RawRow]] +* def parseFileRaw(pathname: String, predicate: Try\[RawRow] => Boolean)(implicit codec: Codec): IO\[Table\[RawRow]] Please note that, in the case of a parameter being an Auto-closeable object such as _InputStream_ or Source, it is the caller's responsibility to close it after parsing. @@ -208,7 +255,7 @@ It is defined thus: val predicate: Try[Row] => Boolean = includeAll def rowParser: RowParser[Row] def builder(rows: Seq[Row]): Table - def parse(ws: Seq[String]): Try[Table] = ... + def parse(ws: Seq[String]): IO[Table] = ... } The type _Row_ defines the specific row type (for example, _Movie_, in the example below). @@ -247,9 +294,9 @@ Typically, the _StandardRowParser_ is used, which takes as its constructor param The methods of _RowParser_ are: - def parse(w: String)(header: Header): Try[Row] + def parse(w: String)(header: Header): IO[Row] - def parseHeader(w: String): Try[Header] + def parseHeader(w: String): IO[Header] ## LineParser @@ -268,9 +315,9 @@ Typically, the _StandardStringsParser_ is used. The methods of _StringsParser_ are: - def parse(ws: Seq[String])(header: Header): Try[Row] + def parse(ws: Seq[String])(header: Header): IO[Row] - def parseHeader(ws: Seq[String]): Try[Header] + def parseHeader(ws: Seq[String]): IO[Header] ## CellParsers @@ -298,6 +345,55 @@ In this case, you must supply a _Map_ which specifies which parser is to be used If the value in that column is not one of the keys of the map, an exception will be thrown. For an example of this, please see the example in _CellParsersSpec_ ("conditionally parse"). +## Content + +The rows of a _Table_ are represented by a case class called _Content_: + + case class Content[+Row](private val xs: ParIterable[Row]) + +Currently, the internal rows are represented by a _ParIterable\[Row]_ which holds the rows in parallel partitions. +This necessarily shuffles the ordering of the rows. + +### Sequence and Sequential + +Tables can be ordered explicitly or they can be ordered by a _Sequence_ member whose values are generated by the parser. +The trait _Sequential_ enables the definition of type constructors which can provide evidence of the +corresponding order. + +For an example of this in use, see the _Crime_ class: + + case class Crime(sequence: Sequence, + maybeCrimeId: Option[BigInt], + month: String, + reportedBy: String, + fallsWithin: String, + maybeLocation: Option[CrimeLocation], + crimeType: String, + lastOutcomeCategory: String, + context: String) extends Sequential + +There is no column in the CSV corresponding to _sequence_. +However, the parser auto-generates that column. + +We can parse the file and write out a one-tenth sample with something like the following: + + import CrimeParser._ + import cats.effect.unsafe.implicits.global + implicit val random: Random = new Random(0) + val sampleFile = "2023-01-metropolitan-street-sample.csv" + val outputFile = "tmp/Crime.use.Resource.csv" + val writeResource = Resource.make(IO(new FileWriter(outputFile)))(fw => IO(fw.close())) + val wi: IO[Unit] = for { + url <- ioResource[Crime](sampleFile) + readResource = Resource.make(IO(Source.fromURL(url)))(src => IO(src.close())) + ct <- readResource.use(src => Table.parseSource(src)) + lt <- IO(ct.mapOptional(m => m.brief)) + st <- IO(lt.filter(FP.sampler(10))) + w <- st.toCSV + _ <- writeResource.use(fw => IO(fw.write(w))) + } yield () + wi.unsafeRunSync() + ## Caveats A case class which represents a row (or part of a row) of the table you want to create from parsing, @@ -313,7 +409,7 @@ In this example, we parse the IMDB Movie dataset from Kaggle. The basic structure of the application code will look something like this: import MovieParser._ - val x: Try[Table[Movie]] = Table.parseResource("movie_metadata.csv") + val x: IO[Table[Movie]] = Table.parseResource("movie_metadata.csv") In this example, the row type is _Movie_, a case class with eleven parameters. The data can be found in a local resource (relative to this class) called movie_metadata.csv. @@ -447,7 +543,7 @@ The example comes from a report on the submissions to a Scala exam. Only one que ) import Submissions._ - val qty: Try[Table[Submission]] = Table.parseSequence(rows) + val qty: IO[Table[Submission]] = Table.parseSequence(rows) Note the use of _cellParserRepetition_. The parameter allows the programmer to define the start value of the sequence number for the columns. In this case, we use the default value: 1 and so don't have to explicitly specify it. @@ -600,7 +696,7 @@ The following example from _JsonRendererSpec.scala_ shows how we can take the fo val strings = List("First, Last", "Adam,Sullivan", "Amy,Avagadro", "Ann,Peterson", "Barbara,Goldman") - val wy: Try[String] = for (pt <- Table.parse[Table[Player]](strings)) yield Player.convertTable(pt).asInstanceOf[Renderable[Partnership]].render + val wy: IO[String] = for (pt <- Table.parse[Table[Player]](strings)) yield Player.convertTable(pt).asInstanceOf[Renderable[Partnership]].render wy should matchPattern { case Success("{\n \"rows\": [{\n \"playerA\": \"Adam S\",\n \"playerB\": \"Amy A\"\n }, {\n \"playerA\": \"Ann P\",\n \"playerB\": \"Barbara G\"\n }],\n \"header\": [\"playerA\", \"playerB\"]\n}") => } implicit val r: JsonFormat[Table[Partnership]] = new TableJsonFormat[Partnership] {} wy.map(p => p.parseJson.convertTo[Table[Partnership]]) should matchPattern { case Success(HeadedTable(_, _)) => } @@ -610,8 +706,9 @@ Release Notes V1.1.2 -> V1.1.3 * Use of Cats IO - [![CircleCI](https://circleci.com/gh/rchillyard/TableParser.svg?style=svg)](https://circleci.com/gh/rchillyard/TableParser) -[CircleCI failure due to missing library] +* Table contents are now parallelized +* Option of having sequential rows of user type +* Improved Analysis by allowing Histogram V1.1.1 -> V1.1.2 * Make RawRow a type (not just a type alias) diff --git a/build.sbt b/build.sbt index 8d7db45b..6ff6d400 100755 --- a/build.sbt +++ b/build.sbt @@ -23,6 +23,8 @@ lazy val nScalaTimeVersion = "2.32.0" lazy val tsecVersion = "0.4.0" libraryDependencies ++= Seq( + "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4", + "org.typelevel" %% "cats-effect" % "3.4.8", "io.github.jmcardon" %% "tsec-cipher-jca" % tsecVersion, "com.phasmidsoftware" %% "flog" % "1.0.8", "io.spray" %% "spray-json" % "1.3.6", @@ -33,6 +35,3 @@ libraryDependencies ++= Seq( "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", "org.scalatest" %% "scalatest" % scalaTestVersion % "test" ) -libraryDependencies += - "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4" - diff --git a/info6205.spring2023.teamproject.csv b/info6205.spring2023.teamproject.csv new file mode 100644 index 00000000..490a0c3c --- /dev/null +++ b/info6205.spring2023.teamproject.csv @@ -0,0 +1,586 @@ +crimeID,longitude,latitude +447a81a19157c2f6ef97accacebaa66d8153e19ca43c16ca452e6d8d447823,-0.009691,51.483548 +112f8b2a663198263314a16a8b52f1f6835cefcbcf0a35388c98ee5db23dd82,-0.118888,51.513075 +1b679ce8cc565f83868ff4a0829af95442b51ffdf4366341a850c6f248f7d41,0.076327,51.540042 +1d2872ccd061abc7b350b54a55a3be5309f19382ccac26d2f4a55e53e3fdde0,-0.418139,51.500839 +28449b49ea4cf6214292dd19df4cf7700fab064cb1be33219eaeef6fbf0e16a,-0.134987,51.46327 +6b745f8b2ac34e26345bccfad2dc9b8901ee2d905e2beab3457cdd366ee93d1,0.063946,51.492689 +6f98975ecfe326d5e13a691623426c80d43b20470dc2d15e62e192fe0567ce6,-0.198751,51.542493 +75d9ab17bbcc3ed268f8f608635ffc63bdb5992566b424d7268e6228cfcee38,0.01742,51.49214 +7b1831dd1e72101f4bd788777c8a7182c5df8d58fc7ef56f4118728cab4bcd6,0.108427,51.575913 +7df64cd68ac7124133805f23cbd44beed1c7abe7caebf32d636def70b8ed4c1,-0.097732,51.559728 +7e4bed22d78d47d660d835fe0f585da5256fae1d49b6f6a3117b4de884f5d8a,-0.195253,51.456633 +7f7192e08b7c098cd315097f114e19f6e53c7a0c415a655d983067bf07a0def,-0.060842,51.620426 +8028e68510231d179b919f2d5c46ee5c59cf0e0916e342c523a3da6882b4b46,-0.395826,51.568804 +82fe0dd600b32e7ec4295c14772991e2dffa2f9e07c162e71b90bf42e96c3a7,0.11003,51.461387 +8f29dc31e94d60f43eebd7006466d1b9cceac90ef434beb939673daf8397f15,-0.316302,51.516965 +8fb428352704b4665200d8406bd71a16689681af18eca9bda99f108db9cec4c,-0.081064,51.570824 +916917fa5d47e29573aa230f3b7d255de462bb01d3dda7691b43d724d026d4a,-0.286444,51.488386 +9571c97f43ea9b59e63be2a0f5d03f8d382bbda7a98a0a4e341b06fd0986e93,-0.412302,51.568875 +9f1ad1fd73bbe58f8ec0f206010c282b58795f3817f68e547639e09910e2d42,-0.173402,51.404326 +a682e196d41c32e7e24da8a50ad9ff9c34b16f4aca486b89fa542a4a9188b5c,-0.14054,51.537349 +a72b315ae3e85f883c28ccf512f6f96ab5ce5b940dd531edab96291d5d31a00,-0.23688,51.480916 +a9293611b8386f5e57c70cc600c6a75bae053ed0dbaaefe07474c399c6bff1e,-0.091246,51.533392 +b48d49d03b0a92ecf71d576368134805faec377cca00040a83f58d421b400a2,0.010867,51.53602 +b4d69ef0aa0b387b1029a07dd94abc05893036d36834fb09e13990f80d56bcd,-0.326938,51.614863 +ca61b73091757c4e024303e24050ac1bc5202337da4bedb0fd83d3626dca598,-0.077598,51.619443 +cb2b0acbe47756621ce929e109ef6812c14d1732638122874a7f3d613cca409,-0.175432,51.515434 +cc14d7da5e2284e4e740cb74a28a9f9272a45aa8a322cafe2d3b4864d2c4459,-0.180597,51.400562 +ccf4e1c29f5674a34accf86c213c362faa8be55e49ec6eb6b9bddeb62a7ac7b,-0.203292,51.485038 +d22d63889b39a9751fec80e1c5719d386cdeafd48df1701ef8c5ed090f328a2,0.104229,51.506217 +d7bc2241dc226478207328d071a7315008359e5541f5dbdb0033e661c1ba74e,-0.254319,51.616224 +e001473c7e564bffc65a6b850ed48b7b760c3a07c51fc1508addd2cab768ff5,-0.142662,51.519946 +e6299675de5708c3807d9c505cf6e088c3acad78b0e0cf9c8e4f804c2b9db58,-0.073278,51.518422 +ee3642671c2d3b8b89b4cc08d30e91712754c2684d94605f67913f5b17f546d,0.085412,51.540987 +f6ed9f5cd8dd88b333c84e68f2c0437f92ce0ff7429bdb6494c560374b10f92,-0.088338,51.505414 +10126803392d956df928e1dbd24e84a31e81c6d3333e25fe7c1d99bdd5c573ab,-0.110317,51.47953 +10418448daacdee2b5f7f8ade02d37ef258ebe2e34275f9f45f6d0751c43a633,0.017404,51.400999 +106a94a7df12169d5220049d9e857d25efa807d7b23398dc48905979b3ec3298,0.219563,51.605036 +10b7c861c676b9e8e68fecbd86020056ed1fffce5f25cbbe3201cafa197d102d,-0.250869,51.509398 +10d68e95731201300a5713bd568333496ceaf568ace90cce0a6dd27416646b61,-0.090684,51.502367 +10e53327cb2ee72f7f6058fa83b1c0190acb0497d55c9e639a21b044f25ae31c,-0.146936,51.509097 +114f0bb6e12e85fec6c26c29d92d05c1cfc2519918190e6addf750c46d4f2703,0.074527,51.558158 +12620ce39b53c7df6d92bbbac391695eceed3ca511e6dd8ed3a3c4e6c9869f18,-0.163434,51.499842 +12b2be0e989567fc48a08538a2cb47698182d408db8c6d3fa1886063787db60f,0.070904,51.560038 +13223ef0434193ae80a22cd31433e9971167cbfe0efd5f491cfdd1429f8e61cc,0.089605,51.57646 +13cdef4fb41c81d56e7ba0c375339c116405a5ceb356063e286833d773931838,0.200686,51.51236 +15d66662c3e448d362d6153f0c34033d4a08f8c1e024dc70a9b5b97e78d6cb16,-0.144307,51.540595 +15f46001bd3950c810ff1f9a71c23b4324731b81d9fa33878805dd6650793619,-0.123461,51.508409 +163f508cb9ef304900f275a667d784f75ee17eadcde04ab1ec46c2340e1adc1f,-0.123056,51.5186 +17bbfbc5231771fab72489cb897c219b374df95752957bf8c5efb2ec567304bf,-0.239714,51.549418 +17ef886579f29a9b3fdc8b505aee2ef0747bcbc2813c73459f49d0317f14dddd,-0.142748,51.539425 +1834720bec978b5c16a0dd4b0dac93b1acc84f9be667cf2b34b9dbc4d53436ec,-0.296968,51.5353 +18b0580f5d467d39fd0c2632505b77a6acfe3674f6579638e753d26558198adb,-0.137787,51.462029 +1930851c21ba64c3b69590f3d62ee6cac12b0d63d662dc4cd150fa3f4f15aad7,0.220916,51.561719 +1a0302e45fae84cccb38bc505a690b29a099481e56ca4b084b06ad61c1c36cae,0.056,51.604769 +1a0583999a0204303777090f73e76fe3eb8a7120c9f6d76af6267a5707afaeaa,-0.045933,51.550118 +1a073ab5cdd7e91eb0e5ffdfad02639a1f8b1eedff83df2333366773e5102057,-0.071852,51.519775 +1aa2013e02300b9bf6ee05670aaa9806dce694c6941d28d90a2af77a2dde0a06,-0.164973,51.525009 +1c3d9a3dc2e67c0929f6f1959c98141c57c1d5f95fffcd3987bd2c824767713f,-0.297572,51.40768 +1c7daffa47b2936d529d13b14e2c1f6eec7578fdc62c9ea280a2de3b107c3faf,-0.227085,51.568193 +1c9f7d756ddb9617c5b8f20bdad21918d90473569faadd823eef60e089e74a15,-0.174833,51.589891 +1cbf2ae8e2c5127593936aedda364153397c45ce9b564124c2129addfbd19cb3,0.007704,51.549348 +1cc7754146657ccfb511845298cbbf3c11296964812b2a5e55f95fe9876bb3a6,-0.021673,51.527833 +1d04970bca67d676e0e9da4ee7643379de4f6bcc4f6973d238cc565fd16987b5,0.043739,51.508231 +1f2cd22ebbee1ed12ae4f310d8a239d3224f053c50787cba46c737c30cdb8ffc,0.118445,51.497058 +1f5650564ae496751a64d1e1a7398e4a4855fa6a4dd852a45fdf9b29192d0c60,-0.461014,51.536406 +1f871994fe79c43af6169feab05ff03ab0e1cc304c1dfd10c267e487fc7e85ee,0.032277,51.607134 +1f982b98681b51ede4aeaf0b8e47725b6bcac414eff0263205a7e9b4a99ceae0,-0.215976,51.44936 +1f9dec2a612a94ab52189510401b5ea6e666b3a164b063ceb95b33ca51bc2856,-0.138028,51.475576 +215d2b6b64740b1fab32b34c61232741e53c7837b619e5e1ebeb0819bae50ed1,-0.048564,51.679836 +220a436069e17fd8bd507b1336d1b93ee59c1b8fb29c7dc6ee6b3dbd2980868c,-0.25759,51.527366 +224764e652cbf2a71a8a0ec515b3b17529005a250a99083775c51f3bc494dffe,0.029116,51.546578 +22995d459e7c83debfd3ce78c2484183ca008200bc44b923bde748f1816ba3ae,-0.020073,51.587939 +22d1bc519a037d1e9dbf95492cc8eee31199a5eab45d247dc5bfb907a84ffb25,-0.105771,51.547152 +233650ef2991156c3ae47413c5b2cf2a39bc11961af21b5a73743f5255f677a4,-0.047461,51.493203 +23a9409899767a4bf468637308089209eaa24ff016fa697235ffb7b2f101ff43,-0.181132,51.54276 +23aca5cede5cd07ae9240463ced9a6e9e12e99e0361edff4cace282df02d2c1d,-0.052682,51.636226 +23be0f37ca10d4a8d7b034ee7361a7e33252fdfae2ba2c94c5fd85ce797c232d,-0.168955,51.473158 +23dccbc83b397ed7ec76d40b8c5eaf29e8da86fd42eefd842a18c23212a38c6e,0.182214,51.577486 +23fd50991eff185642403d91e24e2d07db0dca3019b4844cc0325e472f9a14f0,-0.064836,51.454714 +244d7265ffbae5c9d5bcc50a8ae16ed88cbee64efebdcbbd8b25fc1185734272,-0.011872,51.560526 +252e2e71806aff71108e557c5c81c66b25bd8e8e1c2c17915cf8a3141113db46,-0.082927,51.384626 +2585df4e9ee68534349f2bf43a99ba8c27a79f28023a30ae45e6ede3fb0d2f04,0.125223,51.534355 +25e0e76ed4d71ff3545e9a5786cf719565d83d783cba30a4a1784d102b161a04,-0.167975,51.497252 +276681d9b66bf10d8f852dbd169f5c06c8304b1105d520e77eeadce35715da87,-0.018712,51.58538 +277a23758ad42ec284a9cb21da499de5f0e693f90263959a14e357699c5f4751,-0.13349,51.538028 +278e6fa3c4ee8f70b9a2ae71413d8dbb62898eddfe597058e8ac3b6e2137cd8b,-0.14398,51.576143 +27d01ec1a5a3031aae3b6789b5f59da73a48f4b10796ad3777770f86d3ed5aeb,0.098933,51.48347 +28467f01b0610f6c103d0a69a82c2e960e4f0b9be1e64c9e3f9a4241470ba3e8,-0.215178,51.408918 +28c8dc6efb43f3e529065380899f5e531082ebb957e79211d005ba6b9582d0ae,-0.008424,51.572725 +297d6ce354013e7077ec71908331e79e595fb8eeffe451344c30a743576d8c20,-0.099085,51.601323 +29a5b752f6c01e171c9a353b4b9fe81325ca7513a53bd8c564e4cb02a067dfe2,-0.163368,51.489662 +2a1b26cc704f376f17e829863f2100b9afe5bfb7bcebefa906769cea9db3927f,0.11524,51.527701 +2a5e6e3827392cc6444c4b820a72bc5788dca895c498d0ba57197f0f0decfdb9,0.193726,51.461078 +2a98a3b14e712d60ae9bf77e3f1ee298a9f35a7f7de4bf54d547600c6623f884,-0.20077,51.517255 +2b6284167bddeb2f37c355293f891426733a4e126c588b871cf3ecc6ade8e6f9,-0.325639,51.583803 +2b7e2f79d2120b12c0891cd0e160f01bb63413aa90e19da37f146d6f128d9396,0.108946,51.554352 +2b9294746022ce08892c431fe9a74f5fea221e12adff822a1dd6e8d525179045,-0.271476,51.56381 +2bf3045657eb3ce613db2ae69709fee2c1a59f75c45003fb4982aa6fcaa7fe42,-0.097339,51.48753 +2c04e2a48f13526a9534e27475428d5797f56515bb68a77f4940dbba82aa0647,-0.113994,51.506998 +2c83eb57b8db05d28643830d80226e6c8beed56db99834f12f45e635bf2829fb,0.103109,51.445563 +2c9ff3717311198c87c2032a6826d917f812c3f7dacbe04faec5083d1f2497b9,0.096983,51.402299 +2ca83f3f3f1dc35fd9938d5acaf5e07fdf9ffb58b595491b4d4f9eea06815f40,-0.44738,51.459853 +2cc22220b135156f55d40daa34a1bf5b5b7a7c5f7ea298b55b18ac6cfa7af637,-0.08779,51.656002 +2d8cf538104351ed959b3d228860dd9615583b7826bc4eaad967417572f49fc4,-0.111492,51.490825 +2d93b380bb50eec1eda10ce05a2af56c081f4eb6132e6754a4d3aaadf3450735,-0.138459,51.493621 +2de0cb3d38f40b9c34a5a69ede8807f3902fded1c56a3b48792c291b9be7367c,-0.114785,51.400817 +2e4a774a5da0944a8c226cdb39c922f12b3b8036de5ae0a3471c7c2867574aa7,0.014732,51.48573 +2f0b9c55504ce81290af6c49f78f154bdc59d7db862117420a2370939e6480e3,-0.073932,51.475251 +30242a210964ba37a9a1cf24ca0211f77c1fdaea34975f0ce4593865d207c283,-0.142363,51.513827 +3025b3352b9e59e2210e7adda31521a0bc09da2ceec512f3c3f737280edf1a4e,-0.11047,51.54757 +302837328db997e5bcb3f5bdf49802ec5ccd977c737d7c57506d4ceef887a813,-0.21682,51.44431 +307ae51b6be97fb1acf4ed9293ac03ef31b14bbf34a53630d9d20587f68a6608,-0.140004,51.513708 +30c74704d8d3d5ac9efe976ae469752a47e2eb5814aec07e5989004f5eaf37cc,0.02599,51.509645 +316eaed3af759b8205ff89468c771b261725cb64c7ab4286bab888ddb8cdc716,-0.152167,51.47597 +31ef167a1143039224fc975b71b7f793f05c9d1a9b6f2545f35e6cc2f8cbd42e,-0.140141,51.534753 +320a0b792ec4e6b12957ad8334d8ed973ca959e8943b9c829b50f023c5e58d22,-0.141383,51.549934 +334c78ae034f45396b507d54f6eda218858d022970388f325e5cee3fe6944dd1,-0.145735,51.514195 +338f9cd2f0730c9d3ba886ddd334905cf7884d32fe5d81779626c8ffc44f0ed5,-0.017778,51.50033 +34db30939ab439eadec0a3cb8d6b7e1a3518cae87661695e4ca3dd94a5336c1d,-0.201469,51.577793 +350e2e580a316007b2563c26495ff7d106b18f87b37342c34501a0499431b369,-0.089842,51.526984 +3529eabdc26793bf6f3e3664f1f7170379545b44930ce976195132da5d1d9ff6,-0.140844,51.514684 +36f1d5426ba57613caa2f4747d1aabcbd8b80a0f7f16fd34c8c07b741dcff749,0.001022,51.421945 +37086f41d2f9ed6a382a6894ccb68c2cf205e0cadaa72501cd0c0d476981fc21,-0.107658,51.565977 +378b06a7b27a520b135ad6bd61b483b2dd69e8860616864df695ed7c998ec311,-0.385556,51.473328 +38b586341e1b6d257a0f58e11bed841d388ffcf35d3f8bda62427fc5aea647af,-0.106464,51.530455 +39867f99d64be3df89185d549312247ae2d47596c53a220d177ba357bff9686e,0.104999,51.424692 +39aa3d3003697a22e634941816694f9e893c325961e17a07490c966d39503f24,-0.090905,51.381824 +3a5fa95dcb20b95388974bc2f2ed3baa4aaaee45592aca31d62402036fd4709f,-0.418792,51.504444 +3a935f5c2c34690d392fc88b67e77d32e28b8ef35cd236628891ef873af5fe9b,-0.324134,51.428772 +3abd400cb57e1f766ca08510f4b1382a959a2a6badec409e1d2dd5de163824ba,-0.21842,51.358411 +3ae8b36d00e705f54f0be7d0a33a0b1d68c6d542ac503f2931d2315242acbe21,-0.130895,51.407325 +3b58a6c7068fe09040aefcb8a9051b6db7729f0e781d933f25fb70cfcbdff1b8,-0.124122,51.508087 +3b8ae8a86ac9397540e75954d5ce179e8463efcb6132e543e8966d5e12112325,-0.452096,51.551486 +3bc65672a9745ed841d5de33ddf228a747085a4c65ba8185791fd9bdd75eae5e,-0.120538,51.460693 +3c354e1f1f586582cc03d62d8f1b2d7c084351f94b1b5d94aaf9b2fa286cdfcf,-0.083525,51.58196 +3c55e8a99bf309aaa8995566a6ec97d9c2874da39cf1deedb0ac960e00c8a2d7,-0.094356,51.496888 +3ca55bed04d4515f8064cd2d6d003084c09579528ade2bc84447d8b24cf53257,-0.017873,51.588406 +3cdf3f666851e58caa05da4aa28bdfeb965336cff977b3daa25c4ec93944972e,0.103142,51.44692 +3d5b84f80066ad492c5645a323029b3e793f9be34d95c348f51aafc4b1d6e8c2,-0.068634,51.488158 +3e07b821c08c3600408345d34ab02355a4026cbe31c5ebda6dfd5c0a40d780a4,-0.128701,51.511703 +3e1219106a13a5dd97fb5918b023385d1b7c2d2d61383764a7a49337559c828a,0.020919,51.508069 +3e4a3e5314e1e6fc0cd617d278080b0311337c62cd7800ae663d912d83cea746,-0.024806,51.481394 +4026b5a9eb04353d8389ac005c8ffda778c0be4bc4ced0c9ef8d3142c50f5b0f,-0.034556,51.575467 +403b3551377583f18bc2051096f0e65606a857f99514df82f2a21d4d125c8783,-0.125259,51.514445 +40cd8429a853cd65eaf0d2c7572ce39e875a2971c56bae666876338fc76a47d7,-0.295091,51.570846 +4109ca93b02dfda5d5b08f32f4a201362df9dc8f0351d8c8faaf6cbe20227e0c,-0.480687,51.546608 +41149ceeb60e6f370f2c83fba27fb3eb527e567e303fc52038c540adea82b603,-0.135165,51.646539 +411ed269c947b00ec9c9d161d33b9187fb691cb164bbf99e37cd2b2d5e8e71b5,-0.204648,51.586493 +412a0e25c1392968916d63d8ee40325e5f092fa41aa009b6d8bdc4c8f1ee924d,-0.194289,51.487984 +413b6a591e00f6c15e3eb748714d2a61bae3cedd15a10d4faa8c0f2c12e18474,-0.105349,51.593061 +4151b1e398e976042a4c964096d04df8f9cc475b3101a3947dac406dc6d67403,-0.076587,51.500446 +41f787457a2c38168a822e1a5ca0a6b3eba40d213c4507c049ae0ccca5570cf0,0.045653,51.426741 +4230aa5040f06cb8ab7e5be1b1aab8e39cc2f8b96a4a488decf4697555444234,0.049934,51.421468 +432e29eb31b9d450edf46110633223ea7069332f2dd12ba089013a1d90fc726b,-0.141729,51.545695 +4352ddfca8d8c26a1d9a9073fdc838cc032146395daeebc7b9df14934045f5b4,-0.220797,51.489693 +4387c65cd898b6295bdb034914d1c22d8dbbfe34f4e5e077f5742be434642640,0.12828,51.586251 +4476543ebe8284a7253e9a6266e34f3c49fe398f6553cba0b1f100380b6b50c4,-0.096819,51.527008 +464e9e96898453032317c3d7f7a51ca4911d656dec7cd7f01a380266e1105905,-0.065234,51.492274 +4711700fdfd0bf2d880ab7186af3869383fb2bd56e7427c983abc833cdef98e7,-0.045838,51.538552 +4817437b04235e165c024a68edbfc6b385bc5a25ddba2b02f0f7ca651b55db1a,-0.097211,51.488202 +482f523bd8a12fc167db962a285126aa4aa879eb1551a8f36d998bd286fda011,-0.12833,51.513361 +4a13394d68daeb1fa1693f131f3078aa7a8fb7a43f94b1e3e0e4b6449ee3f416,-0.134995,51.512136 +4a1883ae91a0be1d1fc34a4505f2fbe863d4db3547a007131389db022c480a9e,-0.06317,51.497123 +4a765412591c2b83e7b89f6d0434e30ac89da321f0e3cb59c1aa3280b25f00de,-0.415751,51.527333 +4abe1d1cf969a754f99c51dfc305bc7448098a05ad0f6e1e8447b71f944d70ae,-0.124835,51.44613 +4ba505190585364d965f33a5a4897c27613a7f7f934ef39155b4c2be743b5df9,0.1314,51.495556 +4bdae09b1e9239a9013eedf1ecd4fbb62b8cfbbe9f105aaf11618557f2a5ffeb,-0.315427,51.554333 +4bdbb3cd99c62ffacae6046334bce39e25cd0f2c2971944588978b3841797aa5,-0.132373,51.544881 +4c9b3c399ab3dda64815dd4ca1931eb306444a632a3c06779a7d9dcd8c02bf09,-0.161283,51.493505 +4d06ccce0bd66293dc8ae9f03f69c75a4ccdbab950cad5178e4990d4ef1d3daf,-0.107917,51.5194 +4d2ca0198776cfbd0ce28714b3a56937c0daaddaeb22cb520f36e3dcc6922e48,-0.173905,51.46891 +4d38329ef805013189d8a793b19f14dda26f2e5141c47ed2f8779f3d7913d168,-0.071621,51.572656 +4d485381e4bc29464b7e3363ac67cf05b8c66b36914237a955b225fb35cae1e1,-0.083237,51.524979 +4d6f56f24caf1039c804f64494452f3e41a136871d129788d6700a81c864b883,0.01124,51.406986 +4d80ce2f3c4209a59242c8401dbc94189f26bc8498d17e93ac2903978981eb20,-0.134064,51.510952 +4d9820405ed003fea4931b71fbc96229e1b5c2eb20a033e76097483177b6b010,-0.045371,51.47976 +4e023b96991265ec302d67d632380ba6714b94a16cdbb7fb9f85d160f85fdcf4,-0.031506,51.568231 +4e73681e77902ab22e29496efc3eb3e861028daea3e4d600c290284a7fb31fc0,-0.137787,51.462029 +4e84e99e876292ea0b6dc0e2d6844fde5583a39e49f9c4b34166c32192aa7f6f,-0.122652,51.455502 +4ee86010937ccca2151e4f34117094cac48dd6801600ef83aa2f2dabbb66afd9,0.220417,51.590272 +4f22499a338176d136f831eadd3f51a0433503d64a12a5cbb0a2028491f22e22,-0.167855,51.468986 +4f37cc68aedabd97a444ead2f6c49edaa5c4962574e9e96c6f30ac4262a96dd4,-0.143693,51.530394 +4f67f31e08b8c5000ef5059d76ee2f0828ce70695f4f68af352c97a49f3c3355,-0.131925,51.51179 +4fb86145c1d3c8d688b9da6914f1e27457a52706191ef0c421b6fb69055e9a50,-0.131196,51.510942 +4fe9e53620acf3e434551b3847f81a501650a792e0f2c153da406efa63e292c6,-0.477901,51.543677 +50977127f4c0442fb0e651ff621eb81ff1fd0162dd147b0424613472eae3361b,-0.140778,51.529035 +51337145aa10b55d317849db1d8d61ec7b00c22fe5a14f088cdf816c62faf4ce,-0.075477,51.54558 +51d1c7efc7cf4bfae337b2ee3211ec504f3bbeff9ff407d76a27eefdb1e0428d,-0.047884,51.536887 +52a069b3898e08eec0b9bcb8096313acd95ee197995b1bb0f7a47b423f2d15fd,0.003127,51.454274 +52de235a4f03c076e38ee89642429ec4fac20fa73e04045eecf3e7a14e60598c,-0.204951,51.393796 +5377ec4071bedd8d17004083a1f5cbec36973a15dc625c5e03ff2338fd936aca,-0.131609,51.525296 +53b5c0c1e820a9e094e5654bcbf6d9db769aef9f4138000ca3d9aac9b4f10777,-0.131715,51.513406 +540246be30f9025dd77432ce12863e2da0a850d5c431b688716e81cc292b5bfd,-0.09093,51.52166 +54ffda045822eaafdc8eda556936e58842e7dde0cccc08253f2d892889e87975,-0.306788,51.592759 +55f8739f3c7d0ae37d5997f9e2c648034b3f2061258af27bcc7a6431997cfec3,-0.24128,51.596899 +56b2101cb021c0df6775d4204d6be1317f4d226a136fe5a23f534bb91635b318,-0.09645,51.354449 +56fd56d25defccb51296050ebc6eea588316175c61986473a2177e31dfff0621,0.018628,51.399089 +57bee0d891a4078ddc8a2eeaf48954d703535008f661430c89812302ee2bad43,-0.434272,51.521135 +58afcbec01314900e7e7ecc4ce10190fac95ed6f08043947c9ffc5f88cf09a87,-0.276595,51.46613 +58c759eec1ef9dddb209b764b0fc7b65b40733d11dd66d9ae27ef0776559e51c,0.025958,51.586857 +58f96d5b2c176cdfdf0d7124a0d1c90e1f8ef6c67d0286bad7f9e087c0014b0b,-0.041192,51.636943 +59551a4ca990582da9e61ac7e93acf2d0e457be7c1fb59998b92b37c2f195f4d,-0.18093,51.407393 +595b1a56f32a95e8b83e4531b4bbc3a0f7db9b49553630f1a54ae4bfff2afe1b,0.152121,51.446382 +5a7cf2850b7d8a84e5cf7882efa309e8d8b1c8c8b4f06416cdbff37df6b23239,-0.399098,51.579181 +5acb780e5774d2601f8ce2bef217ebf625a0a217058129caa871cee61c6b4447,0.07574,51.410088 +5add03eb74bd4e84028c57d163594a49597349639468c7135d88f76e73d116db,-0.18196,51.513647 +5b678d439bf693dbd2cb9781269384c21c85fccb36423657413980612103f4fb,-0.192517,51.505474 +5c07d62af681a108b4cb659f4d2cb6b30dbcf50bcc2810ecf143116fa7f85357,0.052438,51.53602 +5d183a2e0a2bf2c34056cac523addec84120e732cf4a7d3f65330bad5d7a7ebe,-0.361011,51.556292 +5e48a1ccd9fa83f12eba59ffb4d4d357003aeecb553ecbdf0ea582887bb9fe9a,-0.34666,51.534988 +5f1c243efb38a1c543633abd0c6a21ba74e748ab0e4f200b96ba28fef264e77b,-0.096036,51.469029 +5f5c03df9cc601cb4650036eec93542e542b243ef09852f72eeb636a7f1594af,0.078675,51.547257 +5fcc48e873d3595283c9eae427d2aa10b49b77e035c8e1c13c7de9ad5eeff988,-0.165692,51.405141 +60b95b84d966be77b6f18fb1611933a022381da11640be5a96ab001a7dafdf68,-0.10296,51.323819 +60f62ec81fe1027b07b05ca367a52c58fe789b7ec2f17afe6234ee59fba71eab,-0.05614,51.529254 +611f24bea2f4dea18312261265e3fcd5d746a24f81db0f7777626bf30e33e026,-0.127464,51.45539 +62c8e447dfb13ce08c0fb69b36462696fb4b5b4aa4962c4214fd6b491451a5df,-0.23052,51.507942 +63199bbdfdc65e4718e83225ea8159cef6c277292f1338ee35d1f602523b71c9,-0.05294,51.554578 +63b352c3d34fface98d271045d48d3b878c7a35aef15553114fd8804d47ead04,-0.144756,51.359704 +63c10dc1395dbcfbf7f56801f33b697c689a75c6b5911092f6909a6e88e04f3f,-0.188065,51.387306 +6403a2220af646cc3fe51b4fdf3ac9dcb926813bd6328212cd73f98db52fd1d4,-0.145742,51.409953 +647a6f762fd73297f5ab862627d49c52dc1a3935d2685f52f5f27d1582635c3d,0.005527,51.435474 +647eb2677337fa90617e59f27ce0ae7b697b2e5a77db1a0dc5099f22e4a9e57f,-0.130689,51.432096 +66c5a1ead66bf1d37b64788ebcd176472b3d68dc27af456086b41942f9fd6ebf,0.196879,51.535373 +672aa40db4cacd4044eb547df86a451e4a19f3310018b3381bae71776dabbb28,-0.417289,51.566119 +68031ec8894f937f6b2e0d952351df17579ca9045df824afc2bd3aebf6b4dd65,-0.198156,51.47611 +6806cec5ed17eb57ef485d7a9ef5c5fba339a1841d9a29650f5e3e7d47735be3,-0.090503,51.477401 +680b665f285e9b955d978ba6e54b10f8562b658228ad2a1b176d5da1c603ce3f,-0.057886,51.518735 +68aacc6d2b9142bfd7bfed8bd25cac4c7cedbb3dcc02c269ff6f21efa6fa7bac,-0.100927,51.396268 +68c849538de69a4e0dec5d9b09c4429cd5d5269bd1ef339c7d7d8a17c6bc7b1e,-0.10254,51.371195 +68fe8858372f760a97b9ae1700f5387a924f5ef2a0ec84057d914808b0a46e26,-0.128136,51.528645 +6928c04808bcce8fec108c37626930d0291bf9277316df6c3f1015091c5e53f8,-0.443468,51.563817 +6a49ccac8c508acb24e193b73ee7351e70eb8d4ccd0fa467e84c78899126ed74,-0.128701,51.511703 +6a9e3166411224c257d37ba992e480099ae84e34be8f41cbd7e96a634e03a16f,-0.250049,51.558988 +6abe7193005979bfce2f13f60f339f81c62a9d846e2c41c3abf0187e984486aa,-0.167888,51.497988 +6ac9f52c9aefe18f4bf1a01f3a33fd0906257c3ed3506a199af912a8e4a63a49,-0.120887,51.510373 +6c269ccd2425d6cdb454da3db1c7eb88ebb611a86a78030e78f27c1f3634dcaf,0.047227,51.58022 +6c3f87c1c0a74e52655952bdc73c7e70a43b52dcad5b91b1d8cbf61d03cafcc4,-0.115479,51.529674 +6cacc8e1c79b959290f06c95360fe72fb24f8b8101884049784a02c5dc4b97cd,-1.15E-4,51.546676 +6d77318822cabf70a7414ab924989e81de5e403f5a0923cda1b09d4c5205042a,-0.136062,51.464088 +6eafbcb60dd441340fb3213a125781e865fdbf34467340aaf71d73b511e454af,-0.140222,51.515088 +6ed4419b2927cd915d5e225b97e32c9bcaf02908340d8465e7bc1b77e66f68b8,-0.006898,51.543293 +6fd853919aef3af5c63d4aa358464011fc9f3b03908c6260ab45d8881154ee9a,-0.385089,51.524856 +70316c1193e2d1936b118e14c007de361a230c378e2971401208bcbda8bc59bd,-0.364328,51.46912 +708fe876d5b9aa29b27d510bdd2e70be5de9b110292960c36ab93d4100444fb2,0.035476,51.608598 +709744fa5a25189b26a4f30469a07238b43e6f32baae7b9b78e88d5d127bebb5,-0.344943,51.57771 +729c694da865783e336720e99cc03692f1f6c5359d58cf5ab84a02df77d8b9eb,-0.100949,51.506374 +731ab45e81bba500a53511dd7730ccdd11f10ac4981d21548aaff35bfe8fd53c,-0.149605,51.491604 +74272c4cd4600ec513dcaf3961e60d9e6d941e2c13768d6d78fe2b671e02972a,-0.104639,51.433396 +743a121c3c3031fddb330c2e8f6a62bfbd22c9e2b35573d63f288528dc9cce32,-0.258056,51.589555 +7475a3e89611e136ca8420f9124a4bb624cdba4d3c78365bd97f328e82fcbe84,-0.133841,51.512513 +74d3d9bae86b1ed45c3932af1b7df8b2126ed973a94754b4c9776ae4de0a8c25,-0.419132,51.594505 +74f268622a490511f0d8ca477beeb1901b5e52bea00a33575d8ca6e1a0abc97f,-0.055552,51.439813 +74fc1859056a95efb4e0a864d1e5ad5f59878a2bd5df264bc005c47860ed972b,-0.292172,51.541678 +75621f45ed72e396250a72987a5ffc4f7a751dc1b47e554ffce741fcf0a3ff9d,-0.192925,51.364989 +771724650c54d47ff50384be4861c19f978eb92625c373a0a3a5599b266ac2e5,-0.135874,51.515684 +77b99291c8e00fd78bdbd294f7439f3e5689ce244b12440be4dc17602c10ee10,-0.483806,51.546064 +77d5ac75cab5110b2646e85fc050b251a282d35152ae4536409960b0d830c956,-0.02639,51.612721 +7829f542f11da0439d251a6bd50a9db6f7a651c6541c32e62bfded788d6faea3,-0.117062,51.336951 +782dc5dad454e1e974b3eb63feba580c69d4046b76bccef7b0f2a5155d60f199,-0.31948,51.510896 +7838369a3b3c3062695c6f9bb03973c1ad5be98b08dc57e7af7641b0ae0bf0de,-0.179462,51.546295 +78ce8cf7642935cddacf649316333040a58787e9f199ff896027f72038cea5e7,-0.127139,51.513243 +793598381f091a67be5519aab00a0b3e2519f4d5b94635fa367058fa69809e85,-0.105962,51.522758 +793c9563b1a27be317ece23ba6232a810fc9eb1b5868642138dd395e049d9298,0.007025,51.630544 +79653e4466600f4cbeb0af8659c1dc739d599036331f5d38b49d627bbc71512a,-0.194685,51.601123 +7982b8e1ec486a52f01cf44821701646ec1df3438ece3ee8c9560d5e25170700,-0.069622,51.456574 +7a86493b2d8129221570692effe8c66fac5b93316b9a37bc8464750f5fd861e4,-0.26804,51.572221 +7a8d2f5270479146d9ed15d6dd130184d3fa0fd33d520798444c1635a24662e9,-0.069253,51.657767 +7b4207c0360cb7c35126dc688791d8736c8389c6b6ba0ae7a2e7fbe25f596939,-0.363082,51.570924 +7b53ece30d1d07f3ae731e8af49e56fdc200a55e68f5447d3e1440171de5b793,-0.068095,51.509785 +7b60616272edf1806edb89927d7be0cdd6a4d34bcf03b7d27fb16b9154e90290,-0.307699,51.602816 +7cc2c259c12d406e1feb9fa4bccc210186d9892873407824f043fd6ff8367275,-0.200903,51.548631 +7d20e251c3db707a6e194f43592f63eb0d2a4fa6d9c7188405a2d45df7e1e779,-0.149304,51.543117 +7d289976785d7b3f2d817ecf85290e082ef7e8c2607e40de3a1be6ca43b604fc,-0.10355,51.453099 +7d40b299d417d7139523af82472864f462254a7a4328dea205ac3bc36f80ad97,-0.39731,51.588571 +7e1ca40df63020982c2578cbc58ec2858cdcc8943837fd59306524bc2e6f47a9,-0.245403,51.558235 +7e4365ab06847f790fc8b35a590fc559154ed00d91ac2f8fd4b55e016cb1f724,-0.287922,51.553863 +7f25aba14114f843525b2b22fb5d92fb6bb5017b5a1dd2b0617bee2396c7b357,-0.297794,51.500377 +7f5867ab97733f2c7159bcc780cb33b6165dd5ba1c68f9637efec44046c59add,-0.034134,51.500588 +7fa7f9eea072030ea8c42f29eabb2c308b2e7c91a6f6cddaa19fb2959c3fe864,-0.416058,51.497717 +7fc50498e28e61134092c5303176506ba4228f8be8ed90958e7d18c2600ec3b1,0.216271,51.609793 +80d45ecde870d05fa93827beab0cabdecdca36cbc35e09607bb0cb90d287be72,-0.141526,51.512078 +80f9a5ec951d1bde604f30a79a4a04be90065502689b35b8063b3754020c80c3,-0.144307,51.540595 +8156c151089788523424065550555be365ef7f2b199b766d43b8464283a6389b,-0.366033,51.442223 +8176c92b9ed73e5a43d70f34176e90da243f65fd30b3ed700bece311df003c99,-0.251808,51.527028 +81c445126ad6b2a916b5d10d9622c4f30e5cd8900b93db31baa9d2dacc0a833d,-0.010768,51.576901 +82590bfadee03fedcc87a8f5c3361ab02498fa5ba77562f7dc829a68e05f2fb7,-0.130917,51.570207 +825fed653b5428297ceb02a80be6af8e033a2bc841034698613c1ad8059dded9,-0.070922,51.610971 +83a96a09d45df6496426a08d01e376ed2d8353a10929ae963fda8f9f79384393,-0.102854,51.528176 +83cdc9340a00b900f84fb16d544fd3f7a7ed4c3434720a394010733e4718e3a4,-0.116432,51.495541 +848c496edfc0a97640af5bc7d015fccb42883a382383d0dab86443ab6958c81e,-0.13493,51.509158 +85591f18a81926c5b05d991b9f78fab15da92eb40a2ae8414dab923ac18d24e7,-0.189797,51.501376 +85c3e3086e39ca8c88bdf8fb7c1b06b42e2dc1d1822405832b25d7845a1372a3,-0.140154,51.516058 +862b4ba95595b091f24c89e058f2d1ac2d6ee8cfc678c6870d4a4b4d812803cc,-0.073055,51.575269 +86bfe1d3cbb26ebc764153aaf869b4a1f25f988e070daa6e84a3fcf534d63cb4,0.033572,51.314603 +86bfec761d62cc604899b960b515d54ba718fc5c9d0930ffefb44d2a109d1ba6,-0.029726,51.650752 +86d09d748a8339521b7ddcb2f0d3d0064e272d7cf4364690d6c6810740399d29,-0.170369,51.50835 +873882f9dcb6b822dfc1a75651778a0501f6d10adaa9aa1079b7e171ab7a38d8,-0.176243,51.492965 +8744192296287152b44d9b2fbccb036ec2fc10b4dacea54f9f5d4feaa31a5dd1,-0.071058,51.488404 +878467d707f80c5ef3aaa3daea00302e2500a0344991e0ea79b817631272ccf2,-0.084836,51.577558 +87987ab287e06a80418125cb98e037c66a4fb1196732e23a253de4cb412f4d9e,-0.165413,51.522669 +8822e70b602fdceb42e6b3f11e0bd6a61a482319cd435a4a0cdff95877b4ab5a,-0.094708,51.497784 +8887c4e13b1a451e7adee4fa819009568e0b882a65383919cb8ce5a546af1507,-0.08245,51.557771 +88ad88645e28e465233a9f1d507295b26be519f7d2930ab31a1ba1d1bfb1c795,-0.458471,51.470361 +88b0e96287b468220538c34fc89b7a77543f574849ac952e91a45fb9cb9bf103,-0.423047,51.523934 +88d22c381c2888aeecb2610f0261643d58e89381d5f87d44aacb677ce2fb9086,0.017226,51.485777 +898f97409f001a8c7936f4f4832c89b5e3b03fddc20b7290e94e8d97fe4d7bfd,0.062405,51.421483 +89b8425140989ce3c1e98ca8cc00f89ba4afdb389940e28b7ece42689f4b51e0,-0.099284,51.52355 +8a17d592158d3e77fa211b8aa26d1ce830afafa70f0fd518d4a5b7c9a3160f63,0.018034,51.405853 +8a1c1ed37db0ad72d576d5a0123ab6822e8bb765a2b0b7590ced4583d0622d14,-0.030302,51.50993 +8b1b88738426f0d438397b9aa7b2e4569b81c38392b21456b453c1b7625cc9f3,0.163625,51.581522 +8b8b188fe0028a53eebe28c392012f5910500c73eb2930947227d4e09f31c7e4,0.030978,51.545323 +8c50de473cf0b08c4bd7369d3f335a81b3fe27add676ef93e24f75dbbb3c0b73,-0.282747,51.535407 +8ce19049dfe019c907ab0d5498cfc09fc9258f5e3b817b265bd19cc9c71916a4,-0.026066,51.47824 +8d557024999d6e4c9edec594162301327b5ea9d0cad300fc71f9613ef884bdb7,0.148079,51.49026 +8fbd6907dd25bac8ca2a22940dfb13e6e10508b7c70014fc43bf20011c843aa8,0.005302,51.622939 +9020f905eafe6132d5157e0e3f50971a675365603e050213920d08d51f84cf86,-0.268458,51.514506 +91497e8042a6fb4a5157e654ce1b559aeb902c3fc36e4ce3703b69fa6eb7a2b1,-0.089168,51.504168 +91654d66cf68ade2467d3c07209e8c0c98a05d5e569bcbc892b13cdeeeacfcbe,-0.06943,51.565102 +9177cf17012928da8f3c4205e5dd3a155b6d14230cff73e0208a5063cc633902,-0.284376,51.559548 +91dcbb437c81725235dea3d5294b3846a133a50564e3ae9d635990ae8804afb3,-0.325683,51.509932 +9250b05b9d0bf010314d99c4c49380cb98e562fd976b8cc6f816e966b87d4299,-0.104245,51.564087 +92708694cede86c1a997826efefabc3a3eb39c6b5caa59637da2d0e3be3b7969,-0.114788,51.462606 +92bc65a1ca83e1d71c0d5c7907b2e015781049dc330f12febe368d14274b465f,-0.255044,51.400022 +92ed031ef6674ba85cb473d34e79193532b3ef83814a545fbad7255a7c37040f,-0.066991,51.54402 +93310f816432be80b92afa025887000ae2ccb4ed2a027dc171b6e5970708064f,0.082697,51.487816 +9388de18964bb8bf15aa55c778c32ef9b8d13ca4370d8be3445b06d33af66ceb,-0.138845,51.565118 +93b257e3782938cd9517c25b1dbd784640dcf4c0b7d759c69989aaf722c9a790,-0.009109,51.462846 +93c29b2d574249d6fa21e84090272ff4360978d7d67dd63c0bb8ff3b23fd0993,-0.351075,51.506716 +93d40f7dafe682ba0255a74f48d46b26953884e505e646689032ea491d5c5f90,-0.072733,51.58423 +9425b72c3c762996ab29e3c28a995d41259d5248ee4072bc32c7e3293bce70a8,-0.092042,51.396996 +95930842d92c2f56d268f6b9a6880023eb334704f14a3a24ffed6d102f7ab447,-0.416454,51.447195 +9595fd75eb6201b71f06592fe3b7eff8158c36e4718ac7f60cd91a54af6f4456,-0.38657,51.507422 +95c04a9e2620d626734b6c21008226f44121b5f1df40387ca6cb043ddb1bc3a7,-0.272681,51.508399 +95caf31e01c90b8adfde80985993c470121897a6459e133e05f72d42aa5c5c16,-0.006898,51.543293 +95e0e8c2572ddefa8f1367ec1dc7d37110120cd693097a25af5d23345706e85a,-0.155941,51.438791 +95eaa2a4bc7bf76a258efb0e95630558a114f9df8afd539d64a8ac3ed303a074,-0.099148,51.560381 +960a2c53a0b8028730c29ae64b8e2e7543581c35b08f2305c4bba7c228eaa7ba,-0.191437,51.489064 +96ba8194f06f240d129ccc608b8457a8cdc1094de7d23d9664288c04d197d25b,-0.051163,51.482392 +974608425be3fbb08db8b5995ac80d5fc471db50f69d15082bcc82a4da6cad0c,-0.148556,51.445662 +9788e27eb4b98052a820d4c3f9d1f8507c4141b8477291dcd417e30f8eb3ffb2,-0.333864,51.597678 +980001e334a3e7a92826211fd6a80f71a7ee5b4a12944da97d5deb97f3ca7ca1,0.077893,51.536435 +9803c3080288537591d6d5f20972d492e374724a1ea2e219ec2edc296b7e8d70,-0.124528,51.51401 +985f670a519c0426ce6b6a2fef0c5b63f9262a68fdc7358e65dece34c800a7a9,-0.02778,51.537288 +98b7dadfdfca9a62c3386565383039efae555c5880e020bec473b2209dd79146,-0.062926,51.470924 +990ed1ececc7011452b9b9d4fb453a22ef8b10f21b027f86353c49af60e29943,-0.080594,51.526132 +999b0a844816d16ddaa787889451715bb37cd8dc3a060badd97fdb5a273049d1,0.183308,51.583216 +9a320d3f07fb4dfbeee67d6d8387184394cb33ce1cc0b44f6862f82226cba542,-0.21435,51.64643 +9a4177be36a8cf545f4082d77e71cfd079e01c34d9610866295117034ea937c0,0.16556,51.54511 +9ab728c57fa32034294e85196d37b056daf4a2f9f42fd22f987bf0d1f1835c5c,0.080044,51.459266 +9c1906beceb3aab2ed5f2b1d678bd48c61ca0db745d2d30ff58076c395715e84,-0.45474,51.535191 +9cc302062d67f45ec44a01deebece28b140f5baba2a6817c91304bfb6a6d8bd1,0.017449,51.573667 +9cf3c893863924094e6ee2a3b1bbaa682f62f99aac0040366648e062b8b8c1e3,-0.061107,51.473079 +9d9f9d4c608502f297000bd84cc558cc3e9fc3038c510fd3f3b737bce4457060,-0.089938,51.505386 +9e113ac9288b326f9a88d53b55ccc05247066789afbe02aa63939abcab2e5200,-0.176258,51.42 +9e2d894d14921131d8f889028b008b7931c2f5f6feb16354b899f8ab419da0ba,-0.099741,51.355152 +9e862b354af5d2711b7d58fbda9b29554bfb363a19aa2f55dd64e7f92d9421bf,-0.099794,51.579491 +9ec8c42e24147e0ee45c89c4837635d45c291e0f279ac798970b1458dfe4b986,-0.185834,51.490506 +9f2992f532c3e90539fc19043045305ce3292d4fc0c2910ac1406e16a6408086,-0.06804,51.498139 +9f37df0b46fa11d02866605e8dd591d48d2a94899f595ea0699a3a2aa51c0da2,-0.099569,51.312091 +9fac4a3e35c15cc6cb5c821646e14ca8849a87688c5df9e41a012f40bca8fb58,-0.129438,51.513064 +9fdd87f2ff80a9a966fc0ab41338b32561ef8653196848819a4578d75aa6b397,-0.128861,51.603411 +a00c7e1bbf6f7e6f5af07d545ef0438259d24674825e4f1fad982c1345bfe7ae,-0.061618,51.661535 +a02fd18efc269366b092a17cf8a2c112ae227a4a747b3677545244af8328d4c3,-0.003886,51.425167 +a0be98ae845886d74256241b07f7d94fac4daa36123fabce70d156d604329847,-0.142205,51.527637 +a1afd5ecfbf471ec06fb50ccd33a87569e17273a72934f12daf28dafc72790fd,0.058852,51.431078 +a1e00b069566865311fb90dce4a68a9fc25f94c42a4fd7b0c18eb6458d8f22f2,0.109061,51.470735 +a2e1b13d32723ddd82a3ca66f41cc9e98acef26fe8005e70434700e1323cd848,-0.397791,51.518214 +a30480d7947d21ea6af6f73e36fecb77e28301bea466dd53b82a1a6e05a64e7a,0.006338,51.487124 +a35b276b674a456938feec3b21e3cc488040ca2610e867f2e3d1f2262ee1a8db,-0.198469,51.459749 +a387d8cee056ade1f45467aed1d76a0061485cf6e78c0a639f9be0baa78b8962,-0.346076,51.507893 +a3f923b712082e36031b43456e9add572365b242f82011a809340b8ad5c7cb4d,-0.135589,51.446032 +a40d97e5ca6a577647295dca899b09105591489c0c839c1590fc79c527cca21a,0.023388,51.473036 +a4c17c8a84284d0284895d0a5e215978e9ad7a66ab29fe481a69e83477a12ddb,-0.110774,51.487289 +a4ca0a4906838bd39ea24085fdf05db3e2292a79666f0447abdb6e4c4adf6f39,-0.042682,51.394167 +a4f3cce4ae30ff4cb62f4cbbacdcd23c3a7bf0d6ec46a374b1dc19776928f6cc,-0.101088,51.372592 +a6159b04325b0514f1d847e07cacb6ed3b181296e341bb0bff0f78ac3ab2cf9b,-0.004595,51.544171 +a6b41a19b0fef5c325b8b647cf7d0c4c9fb59d8f7bec56bea54f1144ca7e0ca5,0.180772,51.579711 +a728aacdd97b99bb4f2f7391476ab99ad5f3d388038ac944c7adb14ab512cc86,-0.241494,51.514824 +a7c1e68b063a47ddecf22f73ae56ba7f26c70e367905df3a15ae2045797dbd67,-0.10553,51.564764 +a802d855e38c47de6cc51bc5dc0b4e5f99e5925a260ca10192c4f413a346742c,-0.099674,51.399107 +a86d5d7080f9c743a8380a683f3c9a925f50271dc458aae5cea9a2082de3849e,-0.299856,51.411621 +a8e19706099726ee476a1900fcd9a6ddbf9e80b0d6e57ba454863e19abd7773f,0.011319,51.524358 +a908485f7aa5eea77165a957aa894d61b63fdaaca8420a0c8476c9c704763f62,-0.093635,51.401212 +a97ee56036894191f0e648e7c53723cc1f75cc0d1b1d82bec7e053f94d6af279,-0.349878,51.531479 +ab1ee72c4504bb598a894fa20054d5f0f0bf9c433aeecf278ab562f11462a25c,-0.010047,51.56367 +ab499dc084ba3e34cfa1cb0615966768217660135e3712c89d628dc154061e92,-0.085508,51.666585 +ab65f6c60f83535dd296653be8123d81e389fc71408a3e0bec0660b6379a661b,-0.115678,51.552087 +abb6a7eeaf20a84d2ae8fe50892b4a82f61b0debc5a99ae7b40b12d9b93e269e,-0.233993,51.504937 +ac1d3dbe3caf6ab362eb7430c4f0a02121f1acecd3f6a7b1a7fa616b688e280f,-0.436794,51.588228 +acbd013889a0c8d3ee30aff285ec4026f93f87bf82af6664595a7fedf4c95ba0,-0.219826,51.503176 +acf45269168df2dcc266307f060d6410ec0926ff3dbfb582a51105f6c88957cf,-0.035466,51.427032 +ae0ce040eed7d2da86e3a4118c17f706cdfa12130616ccd876ab88f977be1b9b,-0.089089,51.60794 +ae2a8ca79b8b7e4b7e16d457d9d5844afedbf9a1a7e110e0a8323b9bb75bb562,-0.097878,51.378988 +ae3dc4fb0f6653c3cbfb64a6c3a35df08f27c7262581f5704abbfd5d8e4e6000,-0.303965,51.375174 +ae52919a850cb2fbe66056cb133dbd566ac21e509a1ba7a9bf5ea8b5e7630704,-0.178057,51.651448 +aeb841b2049b4214dce7a089c122ae9bc992e4a290ffecb973e543eb26de5239,0.145037,51.455522 +aedfd9d4e315122d58d3d7b1259ada2216271fb9d9e15e29ba61ada25e1b560c,-0.069112,51.47072 +aef233218780fa7a789344cca1029ad84caaa6ca0a20c3817696bd168945e408,-0.334538,51.579667 +aef4e58b66cf07b8123de3542bb73b33d6623c8aa3be805f7e596869b8c681a5,-0.131037,51.514492 +afc58c97415a2a07860e6ba7700dc60ce45729c9f44c1f872cca2e207196c267,-0.054353,51.552902 +b0a666fc5b6d37882b225671e3cffbf1025623d89ef163168df47501f2151f69,-0.005327,51.41387 +b0ab195ecf5697a020a94e04fff9298881e60e4ccd586c2ea89a0270d3f8df36,-0.072245,51.491347 +b116f3665388422023d0f4cadaf65ee615a64a5442880a73c00581bc048595f8,-0.076705,51.573918 +b12d43761703022cdc8b34100ca4bf6aa78ab3fbbbf703cc575cf3995ae61f08,-0.333354,51.586772 +b1804953ae0055807f258c955d1d51febbc660c06b00b0b612b7f10f5aff9816,0.208301,51.594358 +b18f63809cd5a7d56b761aa92a3b2a5e325ebffa96e8c29f9584c43f23fbdafb,-0.034319,51.587324 +b1db811ebcc8e9c3947b11429100d83d2a3e447897a7cfdf7697fdb2f3729946,-0.124227,51.485535 +b22c788ff0c84a0b0af53c831cde61e9a5f32d2347ac25f5d034bcea68ec9b4d,-0.204957,51.571318 +b23558be20373f4f073d4cff00e878daf3f2438ac66d06a3f0491213381f3ee1,-0.065772,51.61388 +b38b00b796c9b20acc2eda2a7f6ac4dd51138189706593f6096507c82edf02c2,-0.072296,51.485008 +b42a6e75a28819c75a13cc3c6237d1922891b794cf7e10c5a6b39f239aca517b,-0.193287,51.403237 +b43052ed2f38caddd4fa7991935832ec8d2f604040b2f7311257550db68f5094,-0.279652,51.504266 +b47055eeaaa52b1cf8b8f54c2d9b40e8eda81284047cac5f3fb139f64eac7f7f,-0.088441,51.491926 +b576abde70d40bf86771ef8da1bfd805c8a770f0cb391cc9508ed76d3afd4615,0.004817,51.523687 +b5b6556230378c2e78f3a8a2ae9c438c8cd62624940ab7ea4738fade5470db30,-0.312969,51.556456 +b625981d8a6b51dbbb500e6278301aec2834adc1c4eec4e63013c5b72b736116,-0.487838,51.525812 +b686e61d354a34319c1f930f7f411c1d3d1d7cc9a3a2e22a610c5078611ceab2,-0.209406,51.606807 +b6a69cd4e830965f16c5289df23c9f2dcf513800f5b3b75733c2bf061ccd1eaa,0.045955,51.503993 +b6adc32ca178ec6a4a9767f6d9a91daf943a12dc065a908cd7f5ccd59ef810b6,-0.122473,51.488618 +b733373f449ed4c927624bb7b1028311484af2bae7cd616d270d50df17fccae0,-0.177824,51.482585 +b77d51752f8139c08f87579bf944201cf8001a921634880396d5cfbf2397fcd4,-0.138152,51.523786 +b7c9a6f714ba2820cf845422233eaf8e16983b9b8251b74d9a3bcbc0c9b19308,-0.090273,51.497037 +b8528936760174673bc92d6529ba39147b51f88200e566a408d2bc88524c15c0,-0.09063,51.572571 +b8cd8db74161176bd638ca4616ccec1085f797e5f66d57dcf56904823f776697,-0.045407,51.628983 +b8e2f696b80434f4f2999026227e392517dfd1bef1a02f79a8d5838ab4a9654a,0.080981,51.538044 +b967cbb954e9b42b06035cc64f6aeca6b96c276605d71b2b4b9316699b5b06c7,-0.006314,51.544443 +b97cbecc69cd34bf9a11302782cf7b44c8c98e11bf7ebb7fda82b0fc333aae64,-0.016861,51.523768 +b99dcf0b3d9a63f14d90866d691808aa4ddb172c57d55ebf706ccbda41bc543e,-0.047126,51.467667 +b9c25f49b2b464bfc8085b4e0481038b0e7c622b96802b509f48f6d498fb37e5,0.193274,51.546169 +ba57c8486d64e6f62e42a76a4950c80c014678266cb7e8f4be21d7d71daf068b,-0.142531,51.454757 +baebe2ced20e0abf0316e05895012854cca56971f43bf305a9359490b903b420,-0.14705,51.512336 +bba35131090c9ba135f7927188221dff32f43c1d7a9c4aaec58ba9476bbb1b9b,-0.123456,51.513777 +bbe3107aa79f96386798f3dca5867d093e36b53bc69e94bbf700af024215f5cc,-0.206081,51.51525 +bc1c9fa5e2e5735508523f78bacf6fdfad3c9f120056875e132591d877dd9536,-0.206048,51.59271 +bccb2ce157357ec1c409c3bfb4ed00afc0f67be52c05d6dddf37450f2c757bc7,-0.14171,51.521028 +bcf940491838bb77eafbe0433d8d722ab5aa7ff171edcd448aa94a1b58a15ff6,-0.055438,51.416061 +bdcde9432c53abae6d886b6a94245b84c72c679c062000dd3eedf2b916b9bdee,-0.052301,51.632317 +be6d27fe564f5d535fb735ae38e0235d53f0396121fff30fa9d1e4ea203c80d0,-0.020407,51.445295 +bf2d75e105c7ab5c4f148dda85e6b9691c5655ee78b35784e10e4816f6919884,-0.298226,51.367196 +bf7cda81f3c471218e77e1719518e2618ce7376ae30b171af12ce918251d58f6,-0.03598,51.400629 +bf80906b15bad61b5d8487e03f6c0752d6b9225931b03c21e7e9459682f7d2d7,-0.193474,51.50062 +c00d8b85448b76558c87f67dbd5db94a2294645258243686981387f25d63f894,-0.204348,51.490054 +c06a61a85fe6312d6d24decf6458ad11a86fc61e7dc77829a43a4028cc199bdc,-0.373023,51.483613 +c0adba802b05c4b807c1c502473c4e28170ca3cd0c3c5ed0911c8e6a57b6616d,-0.319555,51.454398 +c0d9bb5eb44817282fd8d52c2a859fb08b9d25c97bef584c3403a884da1ba063,-0.056142,51.53902 +c162910d1ecff656f1becbe9da455148ead283bae5488458013d4b97a2353135,-0.060892,51.551104 +c1639011656d9c0e000cb6480745823898976881fe6c180d0b4d523e56a874e3,-0.129593,51.513848 +c16597233e45ed66d49a3984891ce88ee8df304870033cec8db5ad35f9fe71b5,-0.295363,51.392837 +c17f9fe955c44b383f33d962f7bac8ecfef7b0f61e72694ccd0279e7bbfb1bf9,-0.136902,51.544171 +c1a169aff6aa8bca863c7ec7688f9f836d55cfc42d15920fcfe14ba5597801c9,-0.035686,51.656005 +c1deaa8df5910fd109d7e3005503abbdb8810d899bfcb8432433af48d6157aa7,-0.088734,51.532452 +c2061493aa286ac220d7e9f2f29f72d59760094b8c5417db6ad3b67c48a29645,0.185659,51.579435 +c22d065b5fd2620e659e5c17eae00cc69847d6f54976310c080aeef666262a5f,-0.138734,51.557589 +c3aade6e279ae260d3d2f0b5039635f0b97311ff8aa1abff0ac4012467f8d2d7,-0.052513,51.655359 +c3cb016801acd761912b9c528ebd03a5c86596da513d8c557517ffca7e0b7681,-0.054939,51.54052 +c3ef2e2c1a06930e7e391859871a543f5af3ba0b984cbd6a91f2072ab2f846ba,-0.098947,51.497367 +c42bc721dee86b2d54f57bc9f24c51e4c35668cf2e56e2fbdbf75f5df52e1cd5,-0.316811,51.555153 +c432564feea6c173201e0159e63a8ef41f6baafcdc539777caf5a6d850b0d210,-0.125729,51.528633 +c494b1a861a951e890459e3e8e84f02d05ec45ff08fdfc99a1323b343d193956,0.156162,51.528745 +c5aa82a316f2ea5e3e38670f643561fc19dbccf6f5516de9c480f8eacb00c987,-0.165625,51.49012 +c5f31575455be8c32f56bf3a1b8c726cbfdf92aec28d4b95732f990112e41072,-0.337248,51.443194 +c74de226140614bc8b082f67c396c113c2b288f225b7dbf05053b1989057a64e,-0.361192,51.43917 +c8cf53c76eab9d1c92e74ad01b0d8d3be6a995576a8216062073ff60d533ba9e,-0.11521,51.398513 +c93ea787c5c1e404e1f32e3541bcdf084f97a211591cb0dc63a5a15d17b9410e,-0.143293,51.481028 +c9efc72e3621d1609bb55f0eec112dd79d6a9b08024c12532aa914f8ba6bb937,-0.054824,51.517182 +ca0cceb7c428b05021742a7666cc177b1fc487f96f4dc434d290b7c220ea75a5,-0.131925,51.51179 +ca54b19a04bacb1d0fe1b137fac32c2c47de8b7b04abb05d9bfccdc1e9d9c5a3,-0.136201,51.515087 +ca72443eb621c40c5a3e13baa42bafff200c40584e15aff573332c9576067e12,-0.181478,51.515267 +ca775bf72aa528408ac8a6bb7050514bb669e346fd88f1f12e865496d5f4fa0b,-0.234721,51.51742 +cb39818dff23605d444b54705bbda3cb457751b4fa7fa7b5fd84b6a889ee8c89,-0.007135,51.557658 +cbdcd39e2c67a2090e8393fdc807680b0ab46d7d6b1865bbaeb29d7dbf74a714,-0.266368,51.541065 +cc6d9c8137d0c0181716d238c7e25845ff414ff33c92b86c117de8a0a2c0951a,-0.069615,51.530061 +ccfb88a4a7c731bf9a062b4206f5715e4e0a9201bf28bb37e2967467cb009e90,-0.109663,51.594524 +d04353002b697404173a6606d9ef45b8e3013463f0a54e966a4e2f332e7394fd,-0.16189,51.500987 +d07e62852f3758b7e8a6d282fedf59d3c5a59ea21fdc71c06a490071acbfb049,0.125944,51.498757 +d081f6b76d15fce81d77068220744b2ec5f3f93bb9d7a898846d3d8e8ec0bcd0,0.158537,51.472883 +d10a162f738871f7b7bc8255bbdd1a1ec939c11067e76976c6fd3e1080bf4a14,-0.062738,51.452926 +d11422f44c68757f04e49f8ba1e42e373bfde668f31e255025bf3e067d95bfe3,0.216876,51.506831 +d258500c48e459d7499118c95c94c5db76a44b98fb935cff79c668f5143b61cf,-0.296205,51.393407 +d2ab7fc25a8dec0ed5cc7239a2b11898c05a9f8da02522496ede49fe34f55f0d,-0.417193,51.52274 +d398fae4ebbea0039605a5bb7c34206de400b860dfe1ed130b3d94bcd48a4cdd,-0.154345,51.514052 +d399d09ca411c382223266b81b996bf604b077380ced6369dd0eaa315c895134,-0.189556,51.535076 +d443b1247877129b8a1c31428958032f7e05d9423e0bf851165f9432f1c60f32,0.070904,51.560038 +d4b9dd36f21545fa2372963f8c448f76205969afb61204db09b855d3ecd144e9,-0.128346,51.436987 +d4d1ba9dbc9ce2382dc266bfc205bca67382e3aeb1c6eb20978148a93cc7fa41,-0.111971,51.501201 +d4f8d73496ee577a92da4fa120f5a6beba3f2c78f11b6dc354ba8c4ff1373c88,-0.025574,51.367486 +d53624cddf9a1a017c2d11deb73bcf719bdafcc2f3b3eae72978ac74a6ca7b0f,0.063911,51.582419 +d58e5e50a89b29bbbef743a7a4fd63053f0b622f7c6abe018a07489f39f2d349,-0.086115,51.589026 +d5a5aba27fbcfa38cccd1fd86250c58f795bfe88ac320472ab423b0d5e754e6b,-0.384742,51.510221 +d70c2bfe664ff6eb2b8fa70076656016f82cc0c0e76f4362c6bfbfa2d12ef0b1,-0.066548,51.606501 +d74f6a885b550b9d362c0de40c253e78a8cb75e40a8ffa94bc6523f64aabd449,-0.195055,51.36049 +d7cfb732eea487274bbbf16f782b3ec14b7f7315a140712b829e2fc9fc81f4d0,-0.151474,51.517684 +d7d9cfd69d27b024892018e742f383365fef5658f036657e8a56b3b51a8d7b8b,0.121731,51.492948 +d86b19df10f2a2a74d7b24124a88b25f5868277b79758e6e59b7c8848c2a1727,0.12029,51.546108 +d89f62ab34f98959497dafd00a524cda9651d25de0a1ebef7931bf93eb44521a,-0.366079,51.470774 +d8a5dd73cb8a1aa730e214fd5252912eac7ebe8dd348ddd1d4f58546ecd9be80,-0.152494,51.443638 +d93462af59f93e0afc640a5162e9e6c2e47691de68702af2cfb4f8d31b1fa4e7,0.084536,51.600363 +d96b04a19cd24a4884c18a70abde935689d99e0c276ce330325518380289e912,-0.108195,51.568683 +d9f4771eb38621e60d4d89370ab09f64ac8678b55348c766c82e3830a40b559f,-0.142168,51.359052 +da0bcf80049b3c30aaf091f9bc625a62ca750d116a6d0dc4fe89f99d561fcbbe,0.118688,51.429562 +da0e6ede33eb7823fb6e53b8ad89d0381c68863e952b4365783bd361591703b4,0.101718,51.549652 +da5c139ef2a23f2bbce3480b141ac18995e0556c1146e1f0ac0039fbf5d110be,0.147608,51.550413 +da5cda3048c09e320650998bdca125746cbbf7ebc2cd14a26ae70d9b100ccf12,-0.121916,51.51163 +da6b3e043bbf17b747b4262b77fa0dfff967b41b4c73b9a62f9b8fd78dd98ae6,-0.131196,51.510942 +da6f85f54c5132983256096b9f138fa9fcaa5fe76c1744156e86c7e752c1d259,-0.437115,51.497865 +db6ecb1f3d2f3c6d058a9f8f25f94ec5b0239b47ed8f71b9f11c65bff84551e0,0.083123,51.53002 +db997c7be34d70ed38dc6eaa0da62341ac6fee1d2fd25af5edc7066123a2e6f4,0.009554,51.466898 +db9f2305d0567638805249c204a2a17573d89578f45f3a9b5799fcfff8602fe1,-0.093916,51.468077 +dbae73e0eeecf0084fc85f0358cf82edea84f631237abf9fac64080aad530d1a,-0.068681,51.472736 +dbc76ba0898f1a78e511300230d317933eed49752246f450bddedaac29db82a2,-0.121759,51.602281 +dd28629687f659e6df2e7acffd379f0d814335f60129ee9f7d28bc5ed8be8c5b,-0.123588,51.529849 +dd5a18a180d8975ffdd7a2d1839c4b58f96f6e3a6fd03da19acb44734a42db1d,0.073678,51.55981 +dedc759b65183ff320e282044df9d4cf255c051b6c2694126d8ecda61f642ba6,-0.103875,51.563272 +deea4fe880e288f33ef6d48667d99cb7e975069968ab65607106f8ac407789d9,-0.141993,51.538136 +deefd925d4adc375661457275519693382abf7102c6080effeddc1133478e105,-0.040709,51.542136 +df6c97b7f2411158fd07f65e7592c79a544f5b6077a9108e591c178f0475291f,-0.159461,51.51335 +e01896a9514ea4ee7b6774f3c367885f953d35ac192ddc9fab52f46db5b458d6,-0.083157,51.494268 +e0b4fbba8955d2d801f495ead8ba6439fe60b1d2da58da644ffd0238b05a02b4,-0.023037,51.46836 +e0c9509c3726a4bfbb35dc3faad6ecc053e230624d2fbcc0b45d418194910c55,-0.103398,51.483491 +e137062ba67a97fc7739b7a867259b75132668c5403327f661ee0fd8cd348feb,-0.324351,51.473944 +e193513a6a46e5526197dff3dc72f462a7b75bd9447f6577f0acccfbc9f0ad98,-0.092656,51.526563 +e196bc2c21b56b151b9fb5ea33ae065ee56a57f7b597b7317137a229e236d4ba,0.094587,51.455257 +e228a4a7d5ebcc6441be0fc2d96ef12724f0e86cf99f5a118752b3c7ab76dc46,-0.137324,51.516166 +e2ca3b33a5844c15df1c647d760b37e608f45b6d7cfc0d1333fe6dfc8f40c7b9,-0.142036,51.453122 +e39395195b4170929462d2e214991e9cf9e304c4bbd2112c51dfa1b52706a7d5,0.123764,51.608904 +e395bb787437a25ecf1c874d2c4124dde5d85561035e0d32bae5dd367f3f76fa,-0.196627,51.369471 +e39c30b899b0efdcbb1b5d521274dbb3e4bd2861b599b9cf473852a90b69182f,-0.048247,51.491327 +e425cc37cb9ddbf6e042c70fb9926a2b98a5b6754948b682da09c867b6701c6c,-0.031636,51.520779 +e4d9b4ef0cc8a0ea20e2a4be4a48fc79f92744df69eb6abe9c2555efad678be7,-0.110334,51.381843 +e510c414d29dbb06b4b52fba0ace747503c757e73c2494d13ef9827fc330b647,-0.143774,51.41913 +e5f1290c65f22c4576767183fe595f0e47e944370eb95fde607e5c8677715ede,-0.050053,51.521582 +e72584b1943912a6c733d3ba10bd8331bc4fd266edec19285ea77a9d99673b8a,-0.110415,51.487966 +e7277af90b9789109a59489b42b16aa2c15c31a5975e5ea1f8e68afdbef2cc42,-0.114195,51.489205 +e72856078d6b086aab599d95e84a21a1eb7a63d4d77c424778bafeaa27eba3d9,-0.195551,51.540267 +e76ec56e3effe16a62897eef80e5464c2fd6a921bf787887cf373977e0f527ca,-0.088855,51.394462 +e7c61676b90f83b6282316091e218632ee3a7c8d1ea4ba06629f9f87456d816e,0.154017,51.533407 +e80e0164725204fb143591795d93b12e662ca8e7c786099ca25749ce5d6198db,-0.100995,51.536473 +e9276fe211edb41e03674120dfb568b42a2e58696d851e36158dfadd45bb50ca,-0.359871,51.423432 +ea144ffab1be2dd8c98e058987ada9bd88a465118b0efcce2723b8c236afb56c,-0.04918,51.497809 +ea403dd00881728f7f51c2b34fc9b32b4a99df38e1b1de24bddc46e9e90c51e6,0.050043,51.478192 +ea6b69395e2c79efb93aca721c545f4a6ef2b5954cd92fddd74bb5375765439a,0.005708,51.553159 +eaab9811c6973bf8f7395620d61a8d0808252d6941528601c6ee07ee9f89526d,-0.004623,51.412869 +eb0c3ed22aad820e1dd02c5d6980b9e641559f05d226cbe2d590814d20909250,-0.131436,51.514246 +ec0206c327261ffdd3a62a82295d7336055665b7ec84af315f80f1458ba4fc27,0.181563,51.576028 +ec6093532bc5c22a8f1c0c1a932d49916cdd41397f390d5c4d1461300f27fc2d,-0.111492,51.490825 +ec785f49028ae22ce1f0f0b16ec9d9f0743722094cbe52d313eae2246ad8bdfb,-0.059834,51.590878 +ecd5be4c0ea9950625b2f5cdd7bcdc555550b518dbb8fd1321510d4cbe5ceaeb,0.07958,51.48199 +edb8aa7fb50b472367dd2fd1d013a07439761213b3ae5049fb976c934f0cd05f,0.077682,51.605844 +ee70229cefe10cde1ecf9407c1d5301e65e1e3a8d58c4ed1d27bc209ef492504,-0.22062,51.549724 +eeab5c6310d2795e1186a098e9723aa836805b35e841cf282fda86daa4aeac9b,-0.125347,51.51968 +eeb5d75140c6e80ccd36ee05777f8e54472a71cc505003d4393a6b1517e814da,-0.311227,51.429261 +eeeef6373edeaf23e4dcfd576e1d74b3105081746c4577be30f7941688de228c,-0.0761,51.392517 +ef2afc36376e88725d035f80c5d13a3f12455cca7fae7b227e2a9c561c2b85d9,-0.056118,51.545342 +ef73afd9d7e468d6e9a945f96f157626236feb114732dff5ecf0b33137af384c,-0.036356,51.478737 +ef83901a2a55ed4cad13981f7bf99a23738d40c26af3be7a7b9d29cf0d9395c9,-0.169349,51.462958 +efef0633dc62d87bc12fb7f13168be6c4acbd46817440cd563aa4d065643a496,-0.03933,51.62818 +f009850f39f8a027836555897ac7a48609bb4c503ef612d2ba01dcda42795d5b,-0.102799,51.378124 +f08036ce609c46748652b2a3be82b93f81fdcbf4dffbf9e3111ae33cb8763b73,0.153427,51.561826 +f091838d27c440aaa84c26a74476f8bd0e511a7d303c2a70b70f303db2d90125,-0.16782,51.496827 +f2e22406d554b53fde04ce50de0c1cf55933921a9f04324529643e32abe71547,-0.105847,51.454287 +f3532514d70a487a5fa2608a33f982192968ca20200c34fb5de9586c53b4d2c0,-0.069,51.613718 +f3cf95869adc243a8c11b8bb92a9c5dc612bbeb1ed208a98351de6afdc9f8184,-0.275901,51.563174 +f4e6ce978fc9a82a23784e85ed44f51a0b11f05053a559652ede89c2fa915c85,-0.185383,51.475383 +f5996414c835bce4b16a5a4e83c87812d5adeb910012034e8168d3366e20637b,-0.292869,51.546966 +f5f105be3a1174d5691055c852fcf0e0bbaf48f6fdf3de7fdc4cb3496f67e786,-0.166129,51.553632 +f67c4d861333f675c2838d0c700908b0c842355e0e6cc2e156c6400dd733eee1,-0.037253,51.588911 +f6e4a80fbfa685086f28a1cff24f30aefa2db6be6e6855c525dbc03dfc06b450,0.116274,51.533833 +f75d6cafe891e3a57204513e2b125c120cf27ae6aee0b302d4ed0bb0b4e9302e,-0.164973,51.525009 +f80a5cfa91981a38a1cacd09a24d7cf433e65f9ab4d182da2d01fac7617b46af,0.028645,51.531775 +f8206ba1d4ce4ab5ab8127dc6144b905676c8ff777093ae27223f97304c9b30d,-0.335683,51.581095 +f86271319557bbbdf4598d1fe40a85861916e1a3aaf5c4387630fb589b6ffe9e,-0.09446,51.59382 +f8a06c33377e4e6fdff0e6a1f93ee9fb99f79591a76f30610c10cc52a4a97951,-0.166747,51.443817 +f8ced3c99d494b81067c47364c965c375d3f5065158b33486ea41e460f860d52,-0.080152,51.481773 +f93d765223b26b6145287537c91e12bc4e760bff475fe2e10be10d08f49ba8bd,-0.208901,51.559805 +f95de7def061d2fab1fad5ccc6991d51f30dedde9b905e339750e4e55808d330,-0.101933,51.374063 +f97be9d5c71f849e618eefc6ea3120f3950271c32c1048673531cdb06b75da55,-0.364328,51.46912 +f998ab07d0e9316a4d3d1c36af29db744c1a5981b70fd759158e4b0e9177283d,-0.050022,51.556032 +fadde1aa0aa7975fd59ffc9c86c0372969ba217ebee5ef8f549cf52c9bd83178,-0.036483,51.583008 +fb2f31e4577497723b0afe4f566a4725537f7dc37b35f8187114fe6582faf3c2,-0.093957,51.493347 +fca18db23f01a16ee1267170110d9c63a78ac2b08eb9fbb94c7ffb730cfa30f9,0.192089,51.575535 +fd5efda28344257917aea9985e37863ca826c441669385991142d5e183b5099b,-0.476135,51.49506 +fde14ee09f8b8fcafcdc16a81fd38f20984a4bbef4923c15528f615078d54520,-0.138803,51.513114 +fe8f253cda23c9f2c0a62148289b4eb6aabd51ebb5d131d5dd222fbe8c61eaf0,0.105128,51.411236 +ff0b7ed88414cf97c2b8fd3e1859ee7df652c52927f07922c62e8280653e701b,-0.077528,51.58724 diff --git a/src/it/scala/com/phasmidsoftware/examples/crime/CrimeFuncSpec.scala b/src/it/scala/com/phasmidsoftware/examples/crime/CrimeFuncSpec.scala index 7efa9c8f..ffb502c0 100644 --- a/src/it/scala/com/phasmidsoftware/examples/crime/CrimeFuncSpec.scala +++ b/src/it/scala/com/phasmidsoftware/examples/crime/CrimeFuncSpec.scala @@ -4,29 +4,31 @@ import cats.effect.IO import com.phasmidsoftware.parse.{RawTableParser, TableParser} import com.phasmidsoftware.table.{Analysis, HeadedTable, RawTable, Table} import com.phasmidsoftware.util.EvaluateIO.matchIO -import com.phasmidsoftware.util.FP.resource import com.phasmidsoftware.util.IOUsing import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Seconds, Span} import scala.io.Source +import scala.util.Random class CrimeFuncSpec extends AnyFlatSpec with Matchers { behavior of "Crime" /** - * The following file is ignored for git purposes: + * The file whose filename is Crime.filename is ignored for git purposes: * You need to download and extract it from here: * [[https://www.kaggle.com/datasets/marshuu/crimes-in-uk-2023/download]] + * Once you have downloaded it, remove the first six data rows as these don't seem to belong to the Metropolitan area. + * */ - val crimeFile = "2023-01-metropolitan-street.csv" it should "be ingested and analyzed as a RawTable" in { // Set up the source - val sy: IO[Source] = IO.fromTry(for (u <- resource[CrimeFuncSpec](crimeFile)) yield Source.fromURL(u)) + // NOTE: we specify the complete Metropolitan file (not available on GitHub). + val sy: IO[Source] = for (u <- Crime.ioResourceNotAvailableOnGithub) yield Source.fromURL(u) val fraction = 4 // Set up the parser (we set the predicate only for demonstration purposes) @@ -50,12 +52,12 @@ class CrimeFuncSpec extends AnyFlatSpec with Matchers { import CrimeParser._ - // Create the table - val wsty: IO[Table[Crime]] = Table.parseResource(crimeFile, classOf[CrimeFuncSpec]) + // Create the table + val wsty: IO[Table[Crime]] = Table.parseResource(Crime.filename, classOf[CrimeFuncSpec]) matchIO(wsty, Timeout(Span(60, Seconds))) { case t@HeadedTable(r, _) => - t.size shouldBe 87211 + t.size shouldBe 87205 r take 100 foreach println succeed } @@ -63,26 +65,67 @@ class CrimeFuncSpec extends AnyFlatSpec with Matchers { it should "be ingested and written out properly to CSV" in { import CrimeParser._ - import CrimeRenderer._ - val mti: IO[Table[Crime]] = IOUsing(Source.fromURL(classOf[Crime].getResource(crimeFile)))(x => Table.parseSource(x)) + val mti: IO[Table[Crime]] = IOUsing(Source.fromURL(classOf[Crime].getResource(Crime.filename)))(x => Table.parseSource(x)) val wi: IO[String] = mti flatMap (_.toCSV) matchIO(wi, Timeout(Span(60, Seconds))) { - case w => w should startWith("crimeID,month,reportedBy,fallsWithin,longitude,latitude,location,lsoaCode,lsoaName,crimeType,lastOutcomeCategory,context\n8536e93fb3ce916daa4251bd53c1a4416ba4159a938340be4a7c40cd4873bfcf,2023-01,Metropolitan Police Service,Metropolitan Police Service,-0.681541,50.792113,On or near Fletcher Way,E01031444,Arun 016B,Violence and sexual offences,Under investigation,") + case w => + w.substring(0, 101) shouldBe ",crimeId,month,reportedBy,fallsWithin,location.longitude,location.latitude,location.location,location" + } + } + + it should "create out a sample of brief entries" in { + import CrimeParser._ + implicit val random: Random = new Random(0) + val wi: IO[Table[CrimeBrief]] = for { + ct <- IOUsing(Source.fromURL(classOf[Crime].getResource(Crime.filename)))(x => Table.parseSource(x)) + lt <- IO(ct.filterValid.mapOptional(m => m.brief)) + st <- IO(lt.sample(450)) + } yield st + matchIO(wi, Timeout(Span(10, Seconds))) { + ct => ct.size shouldBe 155 } } it should "be ingested and written out in brief to CSV" in { - import CrimeLocationRenderer._ import CrimeParser._ - val cti: IO[Table[Crime]] = IOUsing(Source.fromURL(classOf[Crime].getResource(crimeFile)))(x => Table.parseSource(x)) + // CONSIDER defining this URL in Crime + val cti: IO[Table[Crime]] = IOUsing(Source.fromURL(classOf[Crime].getResource(Crime.filename)))(x => Table.parseSource(x)) - val wi: IO[String] = for (ct <- cti; lt <- IO(ct.mapOptional(m => m.brief)); _ = println(s"rows: ${lt.size}"); w <- lt.toCSV) yield w + val wi: IO[String] = for { + ct <- cti + lt <- IO(ct.mapOptional(m => m.brief)) + st <- IO(lt.sort.slice(150, 170)) + w <- st.toCSV + } yield w matchIO(wi, Timeout(Span(60, Seconds))) { - case w => w should startWith("crimeID,longitude,latitude\n8536e93fb3ce916daa4251bd53c1a4416ba4159a938340be4a7c40cd4873bfcf,-0.681541,50.792113\n483d52d514591a895c829dece6091c31f797b7dcfd0735ac89685d1d4dabf899,-0.684107,50.780541\n63343c1f1236bad8ce08d130f37760172dc33b20af2b56fafd9189001d014c39,-0.928552,51.923331\na3d980f554d3ece9e8dcda8518ae87bfa9c75d62396105d63fd10390eb7879ed,-0.772051,51.827897\nbfb1d1da32341b7129e789130001d96f7e603088593dc55e30294bc01670ff9e,-0.804965,51.811332\nde18f4ebeefb1d66f3be2c34f1fc056d751d763b57b86c28955ec793d0f77867,0.724588,52.034478\nunidentified,0.140127,51.588913") + case w => + w shouldBe + """crimeID,longitude,latitude + |85b4a97f2b802503658333bff2b1cbb6a85179b3d720b78692feebcf2d63dc,-0.027238,51.474771 + |863604f90d65cdcf5ccb7d864dae9580d8c01be1a73f4415f1254f5dbb493b,-0.452489,51.469799 + |86c3452bc289b73d2d5111165c63242b1e068647ec58fbc88dd8ee6d2f545e,0.121723,51.55056 + |872b7ca64fa7582d3f165bb11af0524ddd3ff24afdf7a90c58662fb9b29049,-0.224735,51.492891 + |87816b5ceefd0bc30a88073ba0f84d9c83279e66892fdb90a31d648b042c00,0.031268,51.477963 + |87f6ca3cad6a4bd66cc395776ec092056ae4ef9d4205eeb658b1d6a484f279,-0.230917,51.546408 + |87f6ca3cad6a4bd66cc395776ec092056ae4ef9d4205eeb658b1d6a484f279,-0.230917,51.546408 + |882ad36f02eb8ed1fdc846f8deeff9f0a0fcfa7ec4de367347e53ba930e6aa,0.051967,51.538681 + |886394cfdc3700537b6ef7e75baec294c57c6eca203bfc824c7b25f4d1510d,-0.084944,51.484289 + |886394cfdc3700537b6ef7e75baec294c57c6eca203bfc824c7b25f4d1510d,-0.084944,51.484289 + |8904d5e3c878c4597d36cf612b0a4dca7e092fab224f22367c0282949e1d6d,-0.286526,51.466599 + |89f7f4c1b6f03ac1a3c36c5ba9f40673a35bcaed46c49e790b9abff529d0fc,-0.062929,51.559519 + |8ab124ca3d2f07f7b4c910c57992a44d918ecd21ae7755a85e407b7b78e122,0.057263,51.606213 + |8ad32137e8bae5a0004dcc76e20c818f12dedce7d03e3df0d4e3b8e7b93d13,0.112912,51.488012 + |8cb06b69ac2aebee7e0340280231a72d5bcfb37d254b7b6a80356f0777ba1f,-0.057831,51.508842 + |8e0b7353d6eff0467607699256e7f68ada36eb7ffcaa82049d299d97b8622d,-0.077877,51.524577 + |8f9321afab6802cd1b6b46ece05c7cd0cb53e1f2bb073cdfc3aeeeb414cbf1,-0.038254,51.437501 + |8fa8b9fd0e95a234069ae923627a4efc20c6f1c921aa738b0007c634e851a0,-0.199476,51.543124 + |902a35564fa1a7a9b2648173055d65d996453d6f48a848a2c5d14b03f71fdd,-0.071621,51.572656 + |929962fbc0f72c0c1449501b56d6fec7905f0cffe85752d6c63acc56bd21a0,-0.115433,51.387509 + |""".stripMargin } } diff --git a/src/it/scala/com/phasmidsoftware/examples/teamproject/ProjectsFuncSpec.scala b/src/it/scala/com/phasmidsoftware/examples/teamproject/ProjectsFuncSpec.scala index 133fc208..96754f3d 100644 --- a/src/it/scala/com/phasmidsoftware/examples/teamproject/ProjectsFuncSpec.scala +++ b/src/it/scala/com/phasmidsoftware/examples/teamproject/ProjectsFuncSpec.scala @@ -240,7 +240,7 @@ class ProjectsFuncSpec extends AnyFlatSpec with Matchers with Futures with Scala implicit val optionStringGenerator: CsvGenerator[Option[String]] = csvGenerators.optionGenerator[String] implicit val teamGenerator: CsvGenerator[Team] = csvGenerators.generator5(Team) implicit val gradeGenerator: CsvGenerator[Grade] = function(csvGenerators) - csvGenerators.generator4(TeamProject) + csvGenerators.generator4(TeamProject.apply) } private def createCsvRendererForTeamProject(function: CsvRenderers => CsvRenderer[Grade]): CsvRenderer[TeamProject] = { @@ -249,6 +249,6 @@ class ProjectsFuncSpec extends AnyFlatSpec with Matchers with Futures with Scala implicit val optionStringRenderer: CsvRenderer[Option[String]] = csvRenderers.optionRenderer[String]() implicit val teamRenderer: CsvRenderer[Team] = csvRenderers.renderer5(Team) implicit val gradeRenderer: CsvRenderer[Grade] = function(csvRenderers) - csvRenderers.renderer4(TeamProject) + csvRenderers.renderer4(TeamProject.apply) } } diff --git a/src/it/scala/com/phasmidsoftware/examples/teamproject/TeamProject.scala b/src/it/scala/com/phasmidsoftware/examples/teamproject/TeamProject.scala index 5d1f921d..ed146a76 100644 --- a/src/it/scala/com/phasmidsoftware/examples/teamproject/TeamProject.scala +++ b/src/it/scala/com/phasmidsoftware/examples/teamproject/TeamProject.scala @@ -5,7 +5,7 @@ package com.phasmidsoftware.examples.teamproject import com.phasmidsoftware.parse._ -import com.phasmidsoftware.table.{Content, HeadedTable, Header, Table} +import com.phasmidsoftware.table._ import java.net.URL /** @@ -31,6 +31,10 @@ import java.net.URL */ case class TeamProject(team: Team, grade: Grade, remarks: String, repository: URL) +object TeamProject { + implicit val orderingTeamProject: Ordering[TeamProject] = NonSequential.ordering[TeamProject, Int](p => p.team.number) +} + case class Team(number: Int, member_1: String, member_2: Option[String], member_3: Option[String], member_4: Option[String]) case class Grade(totalScore: Double, onTime: Double, scopeScale: Double, planningPresentation: Double, presentation: Double, idea: Double, useCases: Double, acceptanceCriteria: Double, teamExecution: Double, code: Double, unitTests: Double, repo: Double) @@ -48,7 +52,7 @@ object TeamProjectParser extends CellParsers { implicit val teamParser: CellParser[Team] = cellParser5(Team) implicit val gradeParser: CellParser[Grade] = cellParser12(Grade) implicit val attributesParser: CellParser[AttributeSet] = cellParser(AttributeSet.apply: String => AttributeSet) - implicit val teamProjectParser: CellParser[TeamProject] = cellParser4(TeamProject) + implicit val teamProjectParser: CellParser[TeamProject] = cellParser4(TeamProject.apply) implicit object TeamProjectConfig extends DefaultRowConfig { override val listEnclosure: String = "" diff --git a/src/it/scala/com/phasmidsoftware/table/AnalysisFuncSpec.scala b/src/it/scala/com/phasmidsoftware/table/AnalysisFuncSpec.scala new file mode 100644 index 00000000..17c1a772 --- /dev/null +++ b/src/it/scala/com/phasmidsoftware/table/AnalysisFuncSpec.scala @@ -0,0 +1,53 @@ +package com.phasmidsoftware.table + +import cats.effect.IO +import com.phasmidsoftware.examples.crime.CrimeLocation +import com.phasmidsoftware.parse.{RawTableParser, TableParser} +import com.phasmidsoftware.table.Column.make +import com.phasmidsoftware.util.EvaluateIO.matchIO +import com.phasmidsoftware.util.FP.{resource, sequence} +import com.phasmidsoftware.util.{EvaluateIO, FP} +import org.scalatest.concurrent.PatienceConfiguration.Timeout +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Seconds, Span} +import scala.io.Source + +class AnalysisFuncSpec extends AnyFlatSpec with Matchers { + implicit val parser: RawTableParser = RawTableParser(TableParser.includeAll, None, forgiving = true).setMultiline(true) + + behavior of "Analysis (functional specs)" + + it should "analyze the complete crime file" in { + val crimeFile = "../examples/crime/2023-01-metropolitan-street.csv" + + implicit object validityRawRow extends Validity[RawRow] { + def isValid(r: RawRow): Boolean = ! { + val latitude: Double = r("latitude").get.toDoubleOption.getOrElse(55) + val longitude: Double = r("longitude").get.toDoubleOption.getOrElse(1) + val lsoaCode = r("LSOA code").getOrElse("") + CrimeLocation.isValid(longitude, latitude, lsoaCode) + } + } + + // Set up the source + val sy: IO[Source] = IO.fromTry(for (u <- FP.resource[Analysis](crimeFile)) yield Source.fromURL(u)) + + val fraction = 1 + // Set up the parser (we set the predicate only for demonstration purposes) + val parser: RawTableParser = RawTableParser().setPredicate(TableParser.sampler(fraction)) + + EvaluateIO.check(parser.parse(sy), Timeout(Span(10, Seconds))) { + case t@HeadedTable(r, _) => + val q = t.filterValid + Analysis(q) match { + case a@Analysis(_, 12, _) => + println(s"Crime analysis: $a") + r take 10 foreach println + case _ => + println(s"Not good analysis") + fail("didnt match") + } + } + } +} diff --git a/src/it/scala/com/phasmidsoftware/table/MovieFuncSpec.scala b/src/it/scala/com/phasmidsoftware/table/MovieFuncSpec.scala index d43c0ab1..47c6d5ff 100644 --- a/src/it/scala/com/phasmidsoftware/table/MovieFuncSpec.scala +++ b/src/it/scala/com/phasmidsoftware/table/MovieFuncSpec.scala @@ -3,9 +3,12 @@ package com.phasmidsoftware.table import cats.effect.IO import cats.effect.unsafe.implicits.global import com.phasmidsoftware.render._ +import com.phasmidsoftware.util.EvaluateIO.matchIO import com.phasmidsoftware.util.IOUsing +import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Seconds, Span} import scala.io.Source class MovieFuncSpec extends AnyFlatSpec with Matchers { @@ -42,9 +45,14 @@ class MovieFuncSpec extends AnyFlatSpec with Matchers { implicit val csvGenerator: CsvGenerator[Movie] = Movie.createMovieCvsGenerator val wi: IO[String] = mti flatMap (_.toCSV) // for (mt <- mti) yield mt.toCSV - wi.unsafeRunSync().startsWith( - """title,format.color,format.language,format.aspectRatio,format.duration,production.country,production.budget,production.gross,production.titleYear,reviews.imdbScore,reviews.facebookLikes,reviews.contentRating.code,reviews.contentRating.age,reviews.numUsersReview,reviews.numUsersVoted,reviews.numCriticReviews,reviews.totalFacebookLikes,director.name.first,director.name.middle,director.name.last,director.name.suffix,director.facebookLikes,actor1.name.first,actor1.name.middle,actor1.name.last,actor1.name.suffix,actor1.facebookLikes,actor2.name.first,actor2.name.middle,actor2.name.last,actor2.name.suffix,actor2.facebookLikes,actor3,genres.xs,plotKeywords.xs,imdb - |Avatar,Color,English,1.78,178,USA,237000000,760505847,2009,7.9,33000,PG,13,3054,886204,723,4834,James,,Cameron,,0,CCH,,Pounder,,1000,Joel,David,Moore,,936,Wes,,Studi,,855,Action,Adventure,Fantasy,Sci-Fi,avatar,future,marine,native,paraplegic,http://www.imdb.com/title/tt0499549/?ref_=fn_tt_tt_1""".stripMargin) shouldBe true + matchIO(wi, Timeout(Span(10, Seconds))) { + w => + w.substring(0, 1000) shouldBe + """title,format.color,format.language,format.aspectRatio,format.duration,production.country,production.budget,production.gross,production.titleYear,reviews.imdbScore,reviews.facebookLikes,reviews.contentRating.code,reviews.contentRating.age,reviews.numUsersReview,reviews.numUsersVoted,reviews.numCriticReviews,reviews.totalFacebookLikes,director.name.first,director.name.middle,director.name.last,director.name.suffix,director.facebookLikes,actor1.name.first,actor1.name.middle,actor1.name.last,actor1.name.suffix,actor1.facebookLikes,actor2.name.first,actor2.name.middle,actor2.name.last,actor2.name.suffix,actor2.facebookLikes,actor3,genres.xs,plotKeywords.xs,imdb + |102 Dalmatians ,Color,English,1.85,100,USA,85000000,66941559,2000,4.8,372,G,,77,26413,84,4182,Kevin,,Lima,,36,Ioan,,Gruffudd,,2000,Eric,,Idle,,795,Jim,,Carter,,439,Adventure,Comedy,Family,dog,parole,parole officer,prison,puppy,http://www.imdb.com/title/tt0211181/?ref_=fn_tt_tt_1 + |13 Hours ,Color,English,2.35,144,USA,50000000,52822418,""".stripMargin + succeed + } } it should "parse and filter the movies from the IMDB dataset" in { diff --git a/src/main/resources/com/phasmidsoftware/table/2023-01-metropolitan-street-sample.csv b/src/main/resources/com/phasmidsoftware/table/2023-01-metropolitan-street-sample.csv index 8837055e..9fca4c65 100644 --- a/src/main/resources/com/phasmidsoftware/table/2023-01-metropolitan-street-sample.csv +++ b/src/main/resources/com/phasmidsoftware/table/2023-01-metropolitan-street-sample.csv @@ -1,10 +1,4 @@ Crime ID,Month,Reported by,Falls within,Longitude,Latitude,Location,LSOA code,LSOA name,Crime type,Last outcome category,Context -8536e93fb3ce916daa4251bd53c1a4416ba4159a938340be4a7c40cd4873bfcf,2023-01,Metropolitan Police Service,Metropolitan Police Service,-0.681541,50.792113,On or near Fletcher Way,E01031444,Arun 016B,Violence and sexual offences,Under investigation, -483d52d514591a895c829dece6091c31f797b7dcfd0735ac89685d1d4dabf899,2023-01,Metropolitan Police Service,Metropolitan Police Service,-0.684107,50.780541,On or near Victoria Road South,E01031437,Arun 017E,Other theft,Investigation complete; no suspect identified, -63343c1f1236bad8ce08d130f37760172dc33b20af2b56fafd9189001d014c39,2023-01,Metropolitan Police Service,Metropolitan Police Service,-0.928552,51.923331,On or near St Marys Close,E01017714,Aylesbury Vale 004D,Violence and sexual offences,Under investigation, -a3d980f554d3ece9e8dcda8518ae87bfa9c75d62396105d63fd10390eb7879ed,2023-01,Metropolitan Police Service,Metropolitan Police Service,-0.772051,51.827897,On or near Restharrow Road,E01017641,Aylesbury Vale 007A,Violence and sexual offences,Under investigation, -bfb1d1da32341b7129e789130001d96f7e603088593dc55e30294bc01670ff9e,2023-01,Metropolitan Police Service,Metropolitan Police Service,-0.804965,51.811332,On or near Walton Grove,E01017637,Aylesbury Vale 017C,Violence and sexual offences,Under investigation, -de18f4ebeefb1d66f3be2c34f1fc056d751d763b57b86c28955ec793d0f77867,2023-01,Metropolitan Police Service,Metropolitan Police Service,0.724588,52.034478,On or near Catesby Meadow,E01029920,Babergh 007H,Violence and sexual offences,Under investigation, ,2023-01,Metropolitan Police Service,Metropolitan Police Service,0.140127,51.588913,On or near Beansland Grove,E01000027,Barking and Dagenham 001A,Anti-social behaviour,, ,2023-01,Metropolitan Police Service,Metropolitan Police Service,0.140194,51.582356,On or near Hatch Grove,E01000027,Barking and Dagenham 001A,Anti-social behaviour,, ,2023-01,Metropolitan Police Service,Metropolitan Police Service,0.135924,51.587353,On or near Gibbfield Close,E01000027,Barking and Dagenham 001A,Anti-social behaviour,, diff --git a/src/main/scala/com/phasmidsoftware/parse/RowParser.scala b/src/main/scala/com/phasmidsoftware/parse/RowParser.scala index 6d8511c4..539e90e0 100644 --- a/src/main/scala/com/phasmidsoftware/parse/RowParser.scala +++ b/src/main/scala/com/phasmidsoftware/parse/RowParser.scala @@ -116,3 +116,10 @@ case class StandardStringsParser[Row: CellParser]() extends StringsParser[Row] { */ def parseHeader(ws: Seq[Strings]): IO[Header] = IO(Header(ws)) } + +trait SelectiveParser[Row, Table] { + + def setForgiving(forgiving: Boolean): TableParser[Table] + + def setPredicate(predicate: Try[Row] => Boolean): TableParser[Table] +} diff --git a/src/main/scala/com/phasmidsoftware/parse/TableParser.scala b/src/main/scala/com/phasmidsoftware/parse/TableParser.scala index 40686b44..f6ad41db 100644 --- a/src/main/scala/com/phasmidsoftware/parse/TableParser.scala +++ b/src/main/scala/com/phasmidsoftware/parse/TableParser.scala @@ -101,10 +101,12 @@ object TableParser { /** * Class to allow the simplification of an expression to parse a source, given a StringTableParser. * - * @param p a StringTableParser. - * @tparam T the underlying type of p (T will be Table[_]). + * CONSIDER should we generalize the type of parser? + * + * @param parser a StringTableParser. + * @tparam T the underlying type of parser (T will be Table[_]). */ - implicit class ImplicitParser[T](p: StringTableParser[T]) { + implicit class ImplicitParser[T](parser: StringTableParser[T]) { /** * Method to parse a IO[Source]. @@ -113,24 +115,24 @@ object TableParser { * @param si a IO[Source]. * @return an IO[T]. */ - def parse(si: IO[Source]): IO[T] = si flatMap doParse + def parse(si: IO[Source]): IO[T] = si flatMap parse /** - * Method to parse an iterator of String. + * Method to parse a Source. + * NOTE the source s will be closed after parsing has been completed (no resource leaks). * - * @param xs an Iterator[String]. + * @param s a Source. * @return an IO[T]. */ - private def doParse(xs: Iterator[String]): IO[T] = p.parse(xs, 1) + def parse(s: Source): IO[T] = IOUsing(s)(x => doParse(x.getLines())) /** - * Method to parse a Source. - * NOTE the source s will be closed after parsing has been completed (no resource leaks). + * Method to parse an iterator of String. * - * @param s a Source. + * @param xs an Iterator[String]. * @return an IO[T]. */ - private def doParse(s: Source): IO[T] = IOUsing(s)(x => doParse(x.getLines())) + private def doParse(xs: Iterator[String]): IO[T] = parser.parse(xs, 1) } val r: Random = new Random() @@ -153,20 +155,16 @@ object TableParser { } /** - * a function which always evaluates as true, regardless of the successfulness of the input. + * A constant function which always evaluates as true, regardless of the successfulness of the input. */ val includeAll: Try[Any] => Boolean = _ => true } -trait CopyableTableParser[Row, Input, Table] { +trait CopyableTableParser[Row, Input, Table] extends SelectiveParser[Row, Table] { def setHeader(header: Header): TableParser[Table] - def setForgiving(forgiving: Boolean): TableParser[Table] - def setMultiline(multiline: Boolean): TableParser[Table] - def setPredicate(predicate: Try[Row] => Boolean): TableParser[Table] - def setRowParser(rowParser: RowParser[Row, Input]): TableParser[Table] } diff --git a/src/main/scala/com/phasmidsoftware/render/CsvGenerators.scala b/src/main/scala/com/phasmidsoftware/render/CsvGenerators.scala index e32fa46f..82b23a5c 100644 --- a/src/main/scala/com/phasmidsoftware/render/CsvGenerators.scala +++ b/src/main/scala/com/phasmidsoftware/render/CsvGenerators.scala @@ -20,7 +20,9 @@ trait CsvGenerators { * @tparam T the underlying type of the first parameter of the input to the render method. * @return a CsvGenerator[ Option[T] ]. */ - def optionGenerator[T](implicit ca: CsvAttributes): CsvGenerator[Option[T]] = new StandardCsvGenerator[Option[T]] + def optionGenerator[T](implicit ca: CsvAttributes): CsvGenerator[Option[T]] = new StandardCsvGenerator[Option[T]] { + override def toColumnName(po: Option[String], name: String): String = super.toColumnName(po, CsvGenerators.stripMaybe(name)) + } /** * Method to return a CsvGenerator[T] which does not output a column header for at all. @@ -410,6 +412,15 @@ trait CsvGenerators { } object CsvGenerators { + + private val regexMaybe = """maybe([A-Z])(\w*)""".r + + def stripMaybe(name: String): String = name match { + case regexMaybe(initial, remainder) => + initial.toLowerCase() + remainder + case x => x + } + implicit object CsvGeneratorBoolean extends StandardCsvGenerator[Boolean] implicit object CsvGeneratorInt extends StandardCsvGenerator[Int] diff --git a/src/main/scala/com/phasmidsoftware/render/CsvRenderer.scala b/src/main/scala/com/phasmidsoftware/render/CsvRenderer.scala index 349f620d..1c20e8ad 100644 --- a/src/main/scala/com/phasmidsoftware/render/CsvRenderer.scala +++ b/src/main/scala/com/phasmidsoftware/render/CsvRenderer.scala @@ -57,7 +57,7 @@ trait BaseCsvRenderer[-T] extends CsvRenderer[T] { */ abstract class ProductCsvRenderer[T <: Product : ClassTag](implicit c: CsvAttributes) extends BaseCsvProductGenerator[T] with BaseCsvRenderer[T] with CsvProduct[T] -abstract class CsvTableRenderer[T: CsvRenderer : CsvGenerator, O: Writable] extends Renderer[Table[T], IO[O]] { +abstract class CsvTableRenderer[T: CsvRenderer : CsvGenerator : Ordering, O: Writable] extends Renderer[Table[T], IO[O]] { /** * Render an instance of T as an O, qualifying the rendering with attributes defined in attrs. @@ -79,7 +79,7 @@ abstract class CsvTableRenderer[T: CsvRenderer : CsvGenerator, O: Writable] exte o => // CONSIDER can remove o2 here and just use o. val o2 = sw.writeRawLine(o)(hdr) - for (r <- x.content.toSeq) yield generateText(sw, tc, o2, r) + for (r <- x.content.ordered) yield generateText(sw, tc, o2, r) o2 } } @@ -103,7 +103,7 @@ abstract class CsvTableRenderer[T: CsvRenderer : CsvGenerator, O: Writable] exte * @param csvAttributes implicit instance of CsvAttributes. * @tparam T the type of object to be rendered, must provide evidence of CsvRenderer[T] amd CsvGenerator[T]. */ -case class CsvTableStringRenderer[T: CsvRenderer : CsvGenerator]()(implicit csvAttributes: CsvAttributes) extends CsvTableRenderer[T, StringBuilder]()(implicitly[CsvRenderer[T]], implicitly[CsvGenerator[T]], Writable.stringBuilderWritable(csvAttributes.delimiter, csvAttributes.quote)) +case class CsvTableStringRenderer[T: CsvRenderer : CsvGenerator : Ordering]()(implicit csvAttributes: CsvAttributes) extends CsvTableRenderer[T, StringBuilder]()(implicitly[CsvRenderer[T]], implicitly[CsvGenerator[T]], implicitly[Ordering[T]], Writable.stringBuilderWritable(csvAttributes.delimiter, csvAttributes.quote)) /** * Case class to help render a Table to a File in CSV format. @@ -114,7 +114,7 @@ case class CsvTableStringRenderer[T: CsvRenderer : CsvGenerator]()(implicit csvA * @param csvAttributes implicit instance of CsvAttributes. * @tparam T the type of object to be rendered, must provide evidence of CsvRenderer[T] amd CsvGenerator[T]. */ -case class CsvTableFileRenderer[T: CsvRenderer : CsvGenerator](file: File)(implicit csvAttributes: CsvAttributes) extends CsvTableRenderer[T, FileWriter]()(implicitly[CsvRenderer[T]], implicitly[CsvGenerator[T]], Writable.fileWritable(file)) +case class CsvTableFileRenderer[T: CsvRenderer : CsvGenerator : Ordering](file: File)(implicit csvAttributes: CsvAttributes) extends CsvTableRenderer[T, FileWriter]()(implicitly[CsvRenderer[T]], implicitly[CsvGenerator[T]], implicitly[Ordering[T]], Writable.fileWritable(file)) /** * Case class to help render a Table to a File in CSV format. @@ -128,7 +128,7 @@ case class CsvTableFileRenderer[T: CsvRenderer : CsvGenerator](file: File)(impli * @tparam T the type of object to be rendered, must provide evidence of CsvRenderer[T] amd CsvGenerator[T]. * @tparam A the cipher algorithm (for which there must be evidence of HexEncryption[A]). */ -case class CsvTableEncryptedFileRenderer[T: CsvRenderer : CsvGenerator : HasKey, A: HexEncryption](file: File)(implicit csvAttributes: CsvAttributes) extends CsvTableRenderer[T, FileWriter]()(implicitly[CsvRenderer[T]], implicitly[CsvGenerator[T]], Writable.fileWritable(file)) { +case class CsvTableEncryptedFileRenderer[T: CsvRenderer : CsvGenerator : Ordering : HasKey, A: HexEncryption](file: File)(implicit csvAttributes: CsvAttributes) extends CsvTableRenderer[T, FileWriter]()(implicitly[CsvRenderer[T]], implicitly[CsvGenerator[T]], implicitly[Ordering[T]], Writable.fileWritable(file)) { override protected def generateText(ow: Writable[FileWriter], tc: CsvRenderer[T], o: FileWriter, t: T): FileWriter = { val key = implicitly[HasKey[T]].key(t) val rendering = tc.render(t, Map()) diff --git a/src/main/scala/com/phasmidsoftware/render/CsvRenderers.scala b/src/main/scala/com/phasmidsoftware/render/CsvRenderers.scala index a7eb608b..062983ef 100644 --- a/src/main/scala/com/phasmidsoftware/render/CsvRenderers.scala +++ b/src/main/scala/com/phasmidsoftware/render/CsvRenderers.scala @@ -5,6 +5,7 @@ package com.phasmidsoftware.render import com.phasmidsoftware.parse.Strings +import com.phasmidsoftware.render.CsvGenerators.stripMaybe import com.phasmidsoftware.table._ import java.net.URL import scala.reflect.ClassTag @@ -122,6 +123,8 @@ trait CsvRenderers { /** * Method to return a CsvRenderer[T] where T is a 2-ary Product and which is based on a function to convert a (P1,P2) into a T. * + * CONSIDER for this and similar methods, reverse the order of rendering the fields and use +: instead of :+ + * * @param construct a function (P1,P2) => T, usually the apply method of a case class. * The sole purpose of this function is for type inference--it is never actually invoked. * @param ca the (implicit) CsvAttributes. @@ -708,6 +711,22 @@ trait CsvRenderers { def toColumnNames(po: Option[String], no: Option[String]): String = new CsvGenerators {}.generator12(construct).toColumnNames(po, no) } + + /** + * Method to return a CsvRenderer[ Option[T] ]. + * + * @param ca the (implicit) CsvAttributes. + * @tparam T the underlying type of the first parameter of the input to the render method. + * @return a CsvRenderer[ Option[T] ]. + */ + def optionProduct[T: CsvRenderer : CsvGenerator](defaultString: String = "")(implicit ca: CsvAttributes): CsvProduct[Option[T]] = new CsvProduct[Option[T]] { + val csvAttributes: CsvAttributes = ca + + def render(to: Option[T], attrs: Map[String, String]): String = (to map (t => implicitly[CsvRenderer[T]].render(t))).getOrElse(defaultString) + + def toColumnName(po: Option[String], name: String): String = + implicitly[CsvGenerator[T]].toColumnName(po, stripMaybe(name)) + } } object CsvRenderers { @@ -723,6 +742,8 @@ object CsvRenderers { implicit object CsvRendererLong extends StandardCsvRenderer[Long] + implicit object CsvRendererBigInt extends StandardCsvRenderer[BigInt] + implicit object CsvRendererDouble extends StandardCsvRenderer[Double] implicit object CsvRendererString extends StandardCsvRenderer[String] diff --git a/src/main/scala/com/phasmidsoftware/table/Analysis.scala b/src/main/scala/com/phasmidsoftware/table/Analysis.scala index 3e445ce4..c5f2e712 100644 --- a/src/main/scala/com/phasmidsoftware/table/Analysis.scala +++ b/src/main/scala/com/phasmidsoftware/table/Analysis.scala @@ -1,18 +1,21 @@ package com.phasmidsoftware.table -import cats.effect.IO import cats.effect.unsafe.implicits.global import com.phasmidsoftware.parse.{RawTableParser, TableParser} -import com.phasmidsoftware.util.FP +import com.phasmidsoftware.table.Statistics.{makeHistogram, makeNumeric} import com.phasmidsoftware.util.FP.sequence +import com.phasmidsoftware.util.{FP, IOUsing} +import java.net.URL +import scala.collection.mutable import scala.io.Source +import scala.util.Try /** * Class to represent the analysis of a table. * * @param rows the number of rows. * @param columns the number of columns. - * @param columnMap a map of column names to Column objects (the statistics of a column). + * @param columnMap a map of column names to Column objects (the analytics of a column). */ case class Analysis(rows: Int, columns: Int, columnMap: Map[String, Column]) { override def toString: String = s"Analysis: rows: $rows, columns: $columns, $showColumnMap" @@ -45,17 +48,21 @@ object Analysis { /** * A representation of the analysis of a column. * - * @param clazz a String denoting which class (maybe which variant of class) this column may be represented as. - * @param optional if true then this column contains nulls (empty strings). - * @param maybeStatistics an optional set of statistics but only if the column represents numbers. + * @param clazz a String denoting which class (maybe which variant of class) this column may be represented as. + * @param optional if true then this column contains nulls (empty strings). + * @param maybeAnalytic an optional Analytic but only if the column represents something which can be analyzed. */ -case class Column(clazz: String, optional: Boolean, maybeStatistics: Option[Statistics]) { +case class Column(clazz: String, optional: Boolean, maybeAnalytic: Option[Analytic]) { override def toString: String = { val sb = new StringBuilder if (optional) sb.append("optional ") sb.append(clazz) - maybeStatistics match { - case Some(s) => sb.append(s" $s") + sb.append(": ") + maybeAnalytic match { + case Some(s) => + sb.append(s"total: ${s.total}") + sb.append("\n") + sb.append(s" $s") case _ => } sb.toString() @@ -87,53 +94,89 @@ object Column { def make(xs: Seq[String]): Option[Column] = { val (ws, nulls) = xs.partition(_.nonEmpty) val nullable: Boolean = nulls.nonEmpty - val co1 = for (xs <- sequence(for (w <- ws) yield w.toIntOption); ys = xs map (_.toDouble)) yield Column("Int", nullable, Statistics.make(ys)) - lazy val co2 = for (xs <- sequence(for (w <- ws) yield w.toDoubleOption); ys = xs) yield Column("Double", nullable, Statistics.make(ys)) - co1 orElse co2 orElse Some(Column("String", nullable, None)) + // CONSIDER we can combine the following two lines + val co1 = for (xs <- sequence(for (w <- ws) yield w.toIntOption); ys = xs map (_.toDouble)) yield Column("Int", nullable, makeNumeric(ys)) + lazy val co2 = for (xs <- sequence(for (w <- ws) yield w.toDoubleOption); ys = xs) yield Column("Double", nullable, makeNumeric(ys)) + lazy val maybeHistogram: Option[Analytic] = makeHistogram(ws) + co1 orElse co2 orElse Some(Column("String", nullable, maybeHistogram)) } } +trait Analytic { + def total: Int +} + /** - * Class to represent the statistics of a column. + * Class to represent the statistics of a numerical column. * * @param mu the mean value. * @param sigma the standard deviation. * @param min the smallest value. * @param max the largest value. */ -case class Statistics(mu: Double, sigma: Double, min: Double, max: Double) { +case class Statistics(total: Int, mu: Double, sigma: Double, min: Double, max: Double) extends Analytic { override def toString: String = s"(range: $min-$max, mean: $mu, stdDev: $sigma)" } +/** + * Case class to represent the histogram of a non-numerical column. + * + * @param keyFreq the key-frequency values. + * @tparam K the key type. + */ +case class Histogram[K](keyFreq: Map[K, Int]) extends Analytic { + def total: Int = keyFreq.values.sum + + override def toString: String = keyFreq.toSeq.sortBy(x => x._2).reverse.map { case (k, n) => s"$k: $n" }.mkString("\n") +} + object Statistics { - def make(xs: Seq[Double]): Option[Statistics] = xs match { + /** + * Make an (optional) Statistics object for a sequence of Double. + * CONSIDER defining the underlying type as a parametric type with context bound Numeric. + * + * @param xs a sequence of Double. + * @return an optional Statistics. + */ + def makeNumeric(xs: Seq[Double]): Option[Statistics] = xs match { case Nil => None - case h :: Nil => Some(Statistics(h, 0, h, h)) - case _ => doMake(xs) + case h :: Nil => Some(Statistics(xs.length, h, 0, h, h)) + case _ => doMakeNumeric(xs) + } + + /** + * Make an (optional) Histogram object for a sequence of String. + * CONSIDER defining the underlying type as a parametric type. + * + * @param xs a sequence of String. + * @return an optional Histogram. + */ + def makeHistogram(xs: Seq[String], ratio: Int = 10): Option[Histogram[String]] = { + val m: mutable.Map[String, Int] = mutable.HashMap[String, Int]() + xs foreach { + x => + val freq = m.getOrElse(x, 0) + m.put(x, freq + 1) + } + if (m.size < xs.size / ratio) Some(Histogram(m.toMap)) + else None } - private def doMake(xs: Seq[Double]): Option[Statistics] = { + private def doMakeNumeric(xs: Seq[Double]): Option[Statistics] = { val mu = xs.sum / xs.size val variance = (xs map (_ - mu) map (x => x * x)).sum / xs.size - Some(Statistics(mu, math.sqrt(variance), xs.min, xs.max)) + Some(Statistics(xs.size, mu, math.sqrt(variance), xs.min, xs.max)) } } object Main extends App { - // TODO merge the two copies of this file into one (it needs to be at the root level of resources) - val crimeFile = "2023-01-metropolitan-street-sample.csv" - - // Set up the source - val sy: IO[Source] = IO.fromTry(for (u <- FP.resource[Analysis](crimeFile)) yield Source.fromURL(u)) - - val fraction = 1 - // Set up the parser (we set the predicate only for demonstration purposes) - val parser: RawTableParser = RawTableParser().setPredicate(TableParser.sampler(fraction)) - - parser.parse(sy).unsafeRunSync() match { - case t@HeadedTable(r, _) => - val analysis = Analysis(t) - println(s"Crime: $analysis") - r take 10 foreach println + // TODO merge the two copies of this sample file into one (it needs to be at the root level of resources) + private val sampleFile = "2023-01-metropolitan-street-sample.csv" + private val triedSampleResource: Try[URL] = FP.resource[Analysis](sampleFile) + private val fraction = 1 + private val parser = RawTableParser().setPredicate(TableParser.sampler(fraction)) + private val ui = IOUsing(for (u <- triedSampleResource) yield Source.fromURL(u)) { + s => parser.parse(s) map (rawTable => println(Analysis(rawTable))) } + ui.unsafeRunSync() } diff --git a/src/main/scala/com/phasmidsoftware/table/Content.scala b/src/main/scala/com/phasmidsoftware/table/Content.scala index 7ea6f14d..a8462e65 100644 --- a/src/main/scala/com/phasmidsoftware/table/Content.scala +++ b/src/main/scala/com/phasmidsoftware/table/Content.scala @@ -1,8 +1,10 @@ package com.phasmidsoftware.table +import com.phasmidsoftware.table.Content.noOrdering +import com.phasmidsoftware.util.FP import scala.collection.parallel.CollectionConverters._ import scala.collection.parallel.ParIterable -import scala.reflect.ClassTag +import scala.util.Random /** * Class to represent the rows of a Table. @@ -11,22 +13,30 @@ import scala.reflect.ClassTag * At present, the rows are implemented as a ParIterable. * However, we might later change the internal representation, thus xs is private. * + * CONSIDER making the private val parameter an Either of ParIterable[Row] or Iterable[Row]. + * That's to say lazy/parallelized vs. eager. + * Take care, however, as both extend GenIterable[Row]. + * + * See [[https://docs.scala-lang.org/overviews/parallel-collections/overview.html]] for more information on parallel collections. + * However, we can note a few things here: + *
    + *
  1. parallel collections remain ordered unless transformed with "bulk" operations such as map, filter;
    2. + *
  2. seq is always an efficient method on parallel collections;
    3. + *
  3. for now, imposition of an explicit ordering is done via sorted or ordered methods.
    4. + *
+ * * @param xs a ParIterable[Row]. * @tparam Row the underlying Row type. */ -case class Content[+Row](private val xs: ParIterable[Row]) { +case class Content[+Row](private val xs: ParIterable[Row]) extends IterableOnce[Row] { def size: Int = xs.size - def knownSize: Int = xs.knownSize - - def toSeq: Seq[Row] = xs.to(Seq) - - def toIndexedSeq: IndexedSeq[Row] = xs.to(IndexedSeq) + def toSeq: Seq[Row] = xs.to(List) - def toArray[B >: Row : ClassTag]: Array[B] = xs.toArray + def toIndexedSeq: IndexedSeq[Row] = xs.toIndexedSeq - def iterator: Iterator[Row] = xs.toIterator + def iterator: Iterator[Row] = xs.iterator def foreach(f: Row => Unit): Unit = xs foreach f @@ -36,15 +46,6 @@ case class Content[+Row](private val xs: ParIterable[Row]) { def map[B](f: Row => B): Content[B] = Content(xs map f) - /** - * This is not, strictly speaking, the correct definition of flatMap for allowing Content to be a monad. - * - * @param f a function of type Row=>ParIterable[B] - * @tparam B the underlying type of the result. - * @return a Content[B]. - */ - def flatMap[B](f: Row => ParIterable[B]): Content[B] = Content(xs flatMap f) - /** * Transform (flatMap) this Table[Row] into a Table[S]. * @@ -55,8 +56,6 @@ case class Content[+Row](private val xs: ParIterable[Row]) { def mapOptional[S](f: Row => Option[S]): Content[S] = Content(for (q <- xs.map(f); r <- q) yield r) - def foldLeft[B](z: B)(op: (B, Row) => B): B = xs.foldLeft(z)(op) - /** * Method to concatenate two Contents. * CONSIDER is this a source of inefficiency? @@ -77,6 +76,26 @@ case class Content[+Row](private val xs: ParIterable[Row]) { def slice(from: Int, until: Int): Content[Row] = Content(xs.slice(from, until)) + /** + * Method to sample from this Content by a deterministic method (every nth row is chosen). + * NOTE: this is not random. + * + * TESTME + * + * @param n the number of rows from which we select the first. + * @return a new Content[Row] with approximately size/n elements. + */ + def step(n: Int): Content[Row] = Content(xs.seq.grouped(n).map(ys => ys.head).toSeq) + + /** + * Method to randomly sample from this Content. + * + * @param n the odds against choosing any particular element. + * @param random an (implicit) Random number generator. + * @return a new Content[Row] with approximately size/n elements. + */ + def sample(n: Int)(implicit random: Random): Content[Row] = filter(FP.sampler(n)) + /** * This should be used only by unit tests and not be code. * @@ -88,13 +107,42 @@ case class Content[+Row](private val xs: ParIterable[Row]) { * Method to transform this Content[Row] into a sorted Content[S] where S is a super-class of Row and for which there is * evidence of Ordering[S]. * + * NOTE that if the specified ordering is noOrdering, then no ordering takes place. + * * @tparam S the underlying type of the resulting Table (a super-type of Row and for which there is evidence of Ordering[S]). * @return a Content[S]. */ - def sorted[S >: Row : Ordering]: Content[S] = Content(toIndexedSeq.map(_.asInstanceOf[S]).sorted) + def sorted[S >: Row : Ordering]: Content[S] = + if (implicitly[Ordering[S]] != noOrdering) + Content(xs.to(IndexedSeq).map(_.asInstanceOf[S]).sorted) + else + this + /** + * Method to transform this Content[Row] into a sorted Seq[S] where S is a super-class of Row and for which there is + * evidence of Ordering[S]. + * + * @tparam S the underlying type of the resulting Table (a super-type of Row and for which there is evidence of Ordering[S]). + * @return a Seq[S]. + */ + def ordered[S >: Row : Ordering]: Seq[S] = + if (implicitly[Ordering[S]] != noOrdering) + xs.to(Seq).map(_.asInstanceOf[S]).sorted + else + xs.to(Seq) } object Content { def apply[T](xs: Iterable[T]): Content[T] = Content(xs.par) + + /** + * Ordering such that all elements appear equal. + * Ideally, this should take linear time for any adaptive sorting method such as Timsort, insertion sort, etc. + * However, within the context of Content, we don't invoke this ordering at all if it is referenced. + * + * @tparam T the underlying type. + * @return an Ordering[T] which always treats everything as the same. + */ + def noOrdering[T]: Ordering[T] = + (_: T, _: T) => 0 } diff --git a/src/main/scala/com/phasmidsoftware/table/Sequential.scala b/src/main/scala/com/phasmidsoftware/table/Sequential.scala new file mode 100644 index 00000000..293fbfda --- /dev/null +++ b/src/main/scala/com/phasmidsoftware/table/Sequential.scala @@ -0,0 +1,150 @@ +package com.phasmidsoftware.table + +import com.phasmidsoftware.parse.CellParser +import com.phasmidsoftware.render.CsvProduct +import com.phasmidsoftware.table.Sequence.SequenceOrdering +import scala.util.{Failure, Success, Try} + +trait Sequential { + val sequence: Sequence +} + +object Sequential { + def ordering[T <: Sequential]: Ordering[T] = (x: T, y: T) => + SequenceOrdering.compare(x.sequence, y.sequence) +} + +class Sequence(val n: Long) extends AnyVal { + def next: Sequence = new Sequence(n + 1) + + override def toString: String = n.toString +} + +object Sequence { + // NOTE this is unashamedly using a var. + // CONSIDER alternative strategies to avoid use of var. + var sequence: Sequence = Sequence(0L) + + def apply(x: Long): Sequence = new Sequence(x) + + // CONSIDER is the following actually necessary? + implicit object SequenceOrdering extends Ordering[Sequence] { + def compare(x: Sequence, y: Sequence): Int = implicitly[Ordering[Long]].compare(x.n, y.n) + } + + implicit object SequenceCellParser extends CellParser[Sequence] { + def convertString(w: String): Try[Sequence] = triedSequence + + def parse(wo: Option[String], row: Row, columns: Header): Try[Sequence] = triedSequence + } + + implicit object SequenceCvsRenderer extends CsvProduct[Sequence] { + val csvAttributes: CsvAttributes = implicitly[CsvAttributes] + + def render(t: Sequence, attrs: Map[String, String]): String = s"#$t" + + def toColumnName(po: Option[String], name: String): String = "" + } + + private def triedSequence = Try { + sequence = sequence.next + sequence + } +} + +object NonSequential { + + /** + * Method to create an Ordering for type T based on an element of type P. + * + * @param f lens function to retrieve a P from a T. + * @tparam T the underlying type of the elements to be ordered. + * @tparam P the underlying type of the key element. + * @return an Ordering[T] + */ + def ordering[T, P: Ordering](f: T => P): Ordering[T] = (x: T, y: T) => + implicitly[Ordering[P]].compare(f(x), f(y)) + + /** + * Method to create an Ordering for type T based on an optional element of type P. + * + * NOTE: this is more complex than it seems to require but if we allow all non-Some/Some cases to return 0, + * we get a Contract exception. + * + * @param f lens function to retrieve an Option[P] from a T. + * @tparam T the underlying type of the elements to be ordered. + * @tparam P the underlying type of the (optional) key element. + * @return an Ordering[T] + */ + def optionalOrdering[T, P: OrderingWithZero](f: T => Option[P]): Ordering[T] = (x: T, y: T) => { + implicit val po = implicitly[OrderingWithZero[P]] // XXX You should ignore the request to add a type annotation here. + (f(x), f(y)) match { + case (Some(a), Some(b)) => + po.compare(a, b) + case (Some(a), None) => + po.compare(a, po.zero) + case (None, Some(b)) => + po.compare(po.zero, b) + case _ => + 0 + } + } + + /** + * Method to create an Ordering for type T based on a tried element of type P. + * See comments on optionalOrdering (above). + * + * @param f lens function to retrieve an Try[P] from a T. + * @tparam T the underlying type of the elements to be ordered. + * @tparam P the underlying type of the (tried) key element. + * @return an Ordering[T] + */ + def tryOrdering[T, P: OrderingWithZero](f: T => Try[P]): Ordering[T] = (x: T, y: T) => { + val po = implicitly[OrderingWithZero[P]] + (f(x), f(y)) match { + case (Success(a), Success(b)) => + po.compare(a, b) + case (Success(a), Failure(_)) => + po.compare(a, po.zero) + case (Failure(_), Success(b)) => + po.compare(po.zero, b) + case _ => + 0 + } + } + +} + +trait OrderingWithZero[X] extends Ordering[X] { + def zero: X +} + +object OrderingWithZero { + + implicit object OrderingWithZeroString extends OrderingWithZero[String] { + def zero: String = "" + + def compare(x: String, y: String): Int = x.compareTo(y) + } + + implicit object OrderingWithZeroBoolean extends OrderingWithZero[Boolean] { + def zero: Boolean = false + + def compare(x: Boolean, y: Boolean): Int = x.compare(y) + } + + /** + * Implicit method to convert an implicit Numeric[X] into an OrderingWithZero[X] for use with + * optionOrdering and tryOrdering methods. + * + * @tparam X the underlying type (must have evidence of Numeric[X]). + * @return an OrderingWithZero[X] + */ + implicit def convert[X: Numeric]: OrderingWithZero[X] = new OrderingWithZeroFromNumeric[X] {} + + private abstract class OrderingWithZeroFromNumeric[X: Numeric] extends OrderingWithZero[X] { + def zero: X = implicitly[Numeric[X]].zero + + def compare(x: X, y: X): Int = implicitly[Numeric[X]].compare(x, y) + } +} \ No newline at end of file diff --git a/src/main/scala/com/phasmidsoftware/table/Table.scala b/src/main/scala/com/phasmidsoftware/table/Table.scala index a009a6d1..6973e410 100644 --- a/src/main/scala/com/phasmidsoftware/table/Table.scala +++ b/src/main/scala/com/phasmidsoftware/table/Table.scala @@ -27,7 +27,7 @@ import scala.util.{Failure, Random, Try} */ trait Table[Row] extends Iterable[Row] { - /** +/** * Optional value of the Header of this Table, if there is one. */ val maybeHeader: Option[Header] @@ -59,7 +59,7 @@ trait Table[Row] extends Iterable[Row] { * @tparam S the type of the rows of the result. * @return a Table[S] which is made up of a concatenation of the results of invoking f on each row this */ - def flatMap[S](f: Row => Iterable[S]): Table[S] = (content map f).foldLeft(unit[S](Nil))((a, e) => a ++ unit(e)) + def flatMap[S](f: Row => Iterable[S]): Table[S] = (content map f).iterator.foldLeft(unit[S](Nil))((a, e) => a ++ unit(e)) /** * Transform (flatMap) this Table[Row] into a Table[S]. @@ -92,12 +92,12 @@ trait Table[Row] extends Iterable[Row] { * Method to generate a Table[S] for a set of rows. * Although declared as an instance method, this method produces its result independent of this. * - * @param sr a Content of S. + * @param sc a Content of S. * @param maybeHeader an optional Header to be used in the resulting Table. * @tparam S the underlying type of the rows and the result. * @return a new instance of Table[S]. */ - def unit[S](sr: Content[S], maybeHeader: Option[Header]): Table[S] + def unit[S](sc: Content[S], maybeHeader: Option[Header]): Table[S] /** * Method to generate a Table[S] for a set of rows. @@ -141,8 +141,11 @@ trait Table[Row] extends Iterable[Row] { /** * Method to select those rows defined by the given range. + * * NOTE: the rows are numbered 1..N. * + * NOTE: unless explicitly ordered, the content might be in random order. + * * @param n the desired row. * @return a new Table[Row] consisting only the row requested. */ @@ -167,7 +170,7 @@ trait Table[Row] extends Iterable[Row] { */ override def toArray[Element >: Row : ClassTag]: Array[Element] = { // XXX huh? - lazy val rs = content.toArray[Element] + lazy val rs = content.iterator.toArray[Element] rs } @@ -255,6 +258,23 @@ trait Table[Row] extends Iterable[Row] { */ override def filterNot(p: Row => Boolean): Table[Row] = processRows(_.filterNot(p)) + /** + * Method to retain only the rows which satisfy the isValid method of ev (i.e. a Validity[Row]). + * + * @param rv (implicit) a Validity[Row]. + * @return Table[Row] consisting only of rows which satisfy Validity. + */ + def filterValid(implicit rv: Validity[Row]): Table[Row] = filter(r => rv.isValid(r)) + + /** + * Method to randomly sample from this Table. + * + * @param n the odds against choosing any particular element. + * @param random an (implicit) Random number generator. + * @return a new Table[Row] with approximately size/n elements. + */ + def sample(n: Int)(implicit random: Random): Table[Row] = processRows(c => c.sample(n)) + /** * slice (as defined by Iterable). * @@ -307,10 +327,11 @@ trait Table[Row] extends Iterable[Row] { * * @param renderer implicit value of CsvRenderer[Row]. * @param generator implicit value of CsvProductGenerator[Row]. + * @param ordering implicit value of Ordering[Row] (apparently not used but I think it is). * @param csvAttributes implicit value of CsvAttributes. * @return a String. */ - def toCSV(implicit renderer: CsvRenderer[Row], generator: CsvGenerator[Row], csvAttributes: CsvAttributes): IO[String] = + def toCSV(implicit renderer: CsvRenderer[Row], generator: CsvGenerator[Row], ordering: Ordering[Row], csvAttributes: CsvAttributes): IO[String] = CsvTableStringRenderer[Row]().render(this) map (_.toString) /** @@ -319,13 +340,14 @@ trait Table[Row] extends Iterable[Row] { * @param file instance of File where the output should be stored. * @param renderer implicit value of CsvRenderer[Row]. * @param generator implicit value of CsvProductGenerator[Row]. + * @param ordering implicit value of Ordering[Row] (apparently not used but I think it is). * @param hasKey implicit value of HasKey[Row]. * This relates to a column which is the "key" column in a CSV (used for identification). * It is not directly related to cryptography. * @tparam A the cipher algorithm (for which there must be evidence of HexEncryption[A]). * @param csvAttributes implicit value of CsvAttributes. */ - def writeCSVFileEncrypted[A: HexEncryption](file: File)(implicit renderer: CsvRenderer[Row], generator: CsvGenerator[Row], hasKey: HasKey[Row], csvAttributes: CsvAttributes): Unit = + def writeCSVFileEncrypted[A: HexEncryption](file: File)(implicit renderer: CsvRenderer[Row], generator: CsvGenerator[Row], ordering: Ordering[Row], hasKey: HasKey[Row], csvAttributes: CsvAttributes): Unit = CsvTableEncryptedFileRenderer[Row, A](file).render(this) /** @@ -336,9 +358,10 @@ trait Table[Row] extends Iterable[Row] { * @param file instance of File where the output should be stored. * @param renderer implicit value of CsvRenderer[Row]. * @param generator implicit value of CsvProductGenerator[Row]. + * @param ordering implicit value of Ordering[Row] (apparently not used but I think it is). * @param csvAttributes implicit value of CsvAttributes. */ - def writeCSVFile(file: File)(implicit renderer: CsvRenderer[Row], generator: CsvGenerator[Row], csvAttributes: CsvAttributes): Unit = + def writeCSVFile(file: File)(implicit renderer: CsvRenderer[Row], generator: CsvGenerator[Row], ordering: Ordering[Row], csvAttributes: CsvAttributes): Unit = CsvTableFileRenderer[Row](file).render(this) /** @@ -347,9 +370,10 @@ trait Table[Row] extends Iterable[Row] { * @param file instance of File where the output should be stored. * @param renderer implicit value of CsvRenderer[Row]. * @param generator implicit value of CsvProductGenerator[Row]. + * @param ordering implicit value of Ordering[Row] (apparently not used but I think it is). * @param csvAttributes implicit value of CsvAttributes. */ - def writeCSVFileIO(file: File)(implicit renderer: CsvRenderer[Row], generator: CsvGenerator[Row], csvAttributes: CsvAttributes): IO[FileWriter] = + def writeCSVFileIO(file: File)(implicit renderer: CsvRenderer[Row], generator: CsvGenerator[Row], ordering: Ordering[Row], csvAttributes: CsvAttributes): IO[FileWriter] = CsvTableFileRenderer[Row](file).render(this) map { f => f.flush(); f } def maybeColumnNames: Option[Seq[String]] = maybeHeader map (_.xs) @@ -580,6 +604,7 @@ object Table { * Method to parse a table from a File as a table of Seq[String]. * * @param f the file. + * @param predicate a predicate which takes a Try[RawRow] and returns a Boolean. * @param maybeFixedHeader an optional fixed header. If None (the default), we expect to find the header defined in the first line of the file. * @param forgiving forcing (defaults to true). If true (the default) then an individual malformed row will not prevent subsequent rows being parsed. * @param codec (implicit) the encoding. @@ -593,8 +618,9 @@ object Table { /** * Method to parse a table from a File as a table of Seq[String]. * - * @param pathname the path name. - * @param codec (implicit) the encoding. + * @param pathname the path name. + * @param predicate a predicate which takes a Try[RawRow] and returns a Boolean. + * @param codec (implicit) the encoding. * @return an IO of Table[RawRow] where RawRow is a Seq[String]. */ def parseFileRaw(pathname: String, predicate: Try[RawRow] => Boolean)(implicit codec: Codec): IO[Table[RawRow]] = { @@ -648,7 +674,7 @@ object Table { * @param csvAttributes implicit value of CsvAttributes. * @return an Iterable[String] */ - def toCSVRow(t: Table[Row])(implicit csvAttributes: CsvAttributes): IO[String] = { + def toCSVRow(t: Table[Row])(implicit ordering: Ordering[Row], csvAttributes: CsvAttributes): IO[String] = { t.maybeHeader match { case Some(hdr) => implicit val z: CsvGenerator[Row] = Row.csvGenerator(hdr) @@ -665,7 +691,7 @@ object Table { * @param csvAttributes implicit value of CsvAttributes. * @return an Iterable[String] */ - def writeCSVFileRow(t: Table[Row], file: File)(implicit csvAttributes: CsvAttributes): IO[FileWriter] = + def writeCSVFileRow(t: Table[Row], file: File)(implicit ordering: Ordering[Row], csvAttributes: CsvAttributes): IO[FileWriter] = t.maybeHeader match { case Some(hdr) => implicit val z: CsvGenerator[Row] = Row.csvGenerator(hdr) @@ -733,13 +759,13 @@ abstract class RenderableTable[Row](rows: Content[Row], val maybeHeader: Option[ * Method to generate a Table[S] for a set of rows. * Although declared as an instance method, this method produces its result independent of this. * - * @param sr a sequence of S. + * @param sc a sequence of S. * @tparam S the underlying type of the rows and the result. * @return a new instance of Table[S]. */ - override def unit[S](sr: Content[S], maybeHeader: Option[Header]): Table[S] = maybeHeader match { - case Some(h) => HeadedTable(sr, h) - case None => UnheadedTable(sr) + override def unit[S](sc: Content[S], maybeHeader: Option[Header]): Table[S] = maybeHeader match { + case Some(h) => HeadedTable(sc, h) + case None => UnheadedTable(sc) } /** @@ -766,8 +792,8 @@ abstract class RenderableTable[Row](rows: Content[Row], val maybeHeader: Option[ // TODO this makes no sense now: the decision is taken inside Content. (if (knownSize1 > -1) rows.toSeq else rows.toSeq) map { case p: Product => ww.writeRow(o2)(p) - case xs: Seq[Row] => ww.writeRowElements(o2)(xs) // TESTME - case xs: Array[Row] => ww.writeRowElements(o2)(xs.toIndexedSeq) // TESTME + case xs: Seq[_] => ww.writeRowElements(o2)(xs) // TESTME + case xs: Array[_] => ww.writeRowElements(o2)(xs.toIndexedSeq) // TESTME case _ => throw TableException("cannot render table because row is neither a Product, nor an array nor a sequence") } o1 @@ -903,6 +929,9 @@ case class HeadedTable[Row](content: Content[Row], header: Header) extends Rende } } + /** + * @return a String representation of this Table + */ override def toString(): String = s"HeadedTable($header) with ${content.size} rows" } diff --git a/src/main/scala/com/phasmidsoftware/table/Validity.scala b/src/main/scala/com/phasmidsoftware/table/Validity.scala new file mode 100644 index 00000000..867e8949 --- /dev/null +++ b/src/main/scala/com/phasmidsoftware/table/Validity.scala @@ -0,0 +1,6 @@ +package com.phasmidsoftware.table + +trait Validity[T] { + + def isValid(t: T): Boolean +} diff --git a/src/main/scala/com/phasmidsoftware/util/FP.scala b/src/main/scala/com/phasmidsoftware/util/FP.scala index d95cbe1d..41b6e128 100644 --- a/src/main/scala/com/phasmidsoftware/util/FP.scala +++ b/src/main/scala/com/phasmidsoftware/util/FP.scala @@ -132,7 +132,7 @@ object FP { /** * Sequence method to combine elements of type Option[X]. - * The result is not defined unless any of the elements are defined. + * The result is not defined unless all of the elements are defined. * * NOTE that the order of the resulting values will be the reverse of the input. * This is for performance reasons. @@ -141,6 +141,7 @@ object FP { * @tparam X the underlying type. * @return an Option of Seq[X]. * NOTE: that the output collection type will be Seq, regardless of the input type + * CONSIDER using Builder */ def sequence[X](xos: Iterable[Option[X]]): Option[Seq[X]] = xos.foldLeft(Option(Seq[X]())) { @@ -191,6 +192,15 @@ object FP { */ def resource[C: ClassTag](resourceName: String): Try[URL] = resourceForClass(resourceName, implicitly[ClassTag[C]].runtimeClass) + /** + * Method to yield a URL for a given resourceForClass in the classpath for C. + * + * @param resourceName the name of the resourceForClass. + * @tparam C a class of the package containing the resourceForClass. + * @return a Try[URL]. + */ + def ioResource[C: ClassTag](resourceName: String): IO[URL] = IO.fromTry(resource(resourceName)) + /** * Method to yield a Try[URL] for a resource name and a given class. * @@ -203,6 +213,15 @@ object FP { case None => Failure(FPException(s"$resourceName is not a valid resource for $clazz")) } + /** + * Method to yield a Try[URL] for a resource name and a given class. + * + * @param resourceName the name of the resource. + * @param clazz the class, relative to which, the resource can be found (defaults to the caller's class). + * @return a Try[URL] + */ + def ioResourceForClass(resourceName: String, clazz: Class[_] = getClass): IO[URL] = IO.fromTry(resourceForClass(resourceName, clazz)) + /** * Method to determine if the String w was found at a valid index (i). * diff --git a/src/test/resources/com/phasmidsoftware/examples/crime/2023-01-metropolitan-street-sample.csv b/src/test/resources/com/phasmidsoftware/examples/crime/2023-01-metropolitan-street-sample.csv index 8837055e..9fca4c65 100644 --- a/src/test/resources/com/phasmidsoftware/examples/crime/2023-01-metropolitan-street-sample.csv +++ b/src/test/resources/com/phasmidsoftware/examples/crime/2023-01-metropolitan-street-sample.csv @@ -1,10 +1,4 @@ Crime ID,Month,Reported by,Falls within,Longitude,Latitude,Location,LSOA code,LSOA name,Crime type,Last outcome category,Context -8536e93fb3ce916daa4251bd53c1a4416ba4159a938340be4a7c40cd4873bfcf,2023-01,Metropolitan Police Service,Metropolitan Police Service,-0.681541,50.792113,On or near Fletcher Way,E01031444,Arun 016B,Violence and sexual offences,Under investigation, -483d52d514591a895c829dece6091c31f797b7dcfd0735ac89685d1d4dabf899,2023-01,Metropolitan Police Service,Metropolitan Police Service,-0.684107,50.780541,On or near Victoria Road South,E01031437,Arun 017E,Other theft,Investigation complete; no suspect identified, -63343c1f1236bad8ce08d130f37760172dc33b20af2b56fafd9189001d014c39,2023-01,Metropolitan Police Service,Metropolitan Police Service,-0.928552,51.923331,On or near St Marys Close,E01017714,Aylesbury Vale 004D,Violence and sexual offences,Under investigation, -a3d980f554d3ece9e8dcda8518ae87bfa9c75d62396105d63fd10390eb7879ed,2023-01,Metropolitan Police Service,Metropolitan Police Service,-0.772051,51.827897,On or near Restharrow Road,E01017641,Aylesbury Vale 007A,Violence and sexual offences,Under investigation, -bfb1d1da32341b7129e789130001d96f7e603088593dc55e30294bc01670ff9e,2023-01,Metropolitan Police Service,Metropolitan Police Service,-0.804965,51.811332,On or near Walton Grove,E01017637,Aylesbury Vale 017C,Violence and sexual offences,Under investigation, -de18f4ebeefb1d66f3be2c34f1fc056d751d763b57b86c28955ec793d0f77867,2023-01,Metropolitan Police Service,Metropolitan Police Service,0.724588,52.034478,On or near Catesby Meadow,E01029920,Babergh 007H,Violence and sexual offences,Under investigation, ,2023-01,Metropolitan Police Service,Metropolitan Police Service,0.140127,51.588913,On or near Beansland Grove,E01000027,Barking and Dagenham 001A,Anti-social behaviour,, ,2023-01,Metropolitan Police Service,Metropolitan Police Service,0.140194,51.582356,On or near Hatch Grove,E01000027,Barking and Dagenham 001A,Anti-social behaviour,, ,2023-01,Metropolitan Police Service,Metropolitan Police Service,0.135924,51.587353,On or near Gibbfield Close,E01000027,Barking and Dagenham 001A,Anti-social behaviour,, diff --git a/src/test/scala/com/phasmidsoftware/examples/crime/Crime.scala b/src/test/scala/com/phasmidsoftware/examples/crime/Crime.scala index 1cc84b58..47abd7f3 100644 --- a/src/test/scala/com/phasmidsoftware/examples/crime/Crime.scala +++ b/src/test/scala/com/phasmidsoftware/examples/crime/Crime.scala @@ -1,122 +1,212 @@ package com.phasmidsoftware.examples.crime +import cats.effect.IO +import com.phasmidsoftware.examples.crime.CrimeLocation.camelToSnakeCaseColumnNameMapper import com.phasmidsoftware.parse._ import com.phasmidsoftware.render._ import com.phasmidsoftware.table._ -import scala.util.Try +import com.phasmidsoftware.util.FP.ioResource +import com.phasmidsoftware.util.{EvaluateIO, IOUsing} +import java.net.URL +import org.scalatest.concurrent.PatienceConfiguration.Timeout +import org.scalatest.time.{Seconds, Span} +import scala.io.Source +import scala.util.{Random, Try} /** - * This example of table parsing is based on the Kaggle data set: + * Case class to represent a Crime from the Kaggle data set: * [[https://www.kaggle.com/datasets/marshuu/crimes-in-uk-2023?select=2023-01-metropolitan-street.csv]] * - * The file under resources is an edited version of the Metropolitan Crime Statistics 2023-01 (only the first 5,000 rows) + * The file under main/resources is an edited version of the Metropolitan Crime Statistics 2023-01 (only the first 5,000 rows) * - * @param crimeID (optional BigInt in hexadecimal notation) see Kaggle. + * @param maybeCrimeId (optional BigInt in hexadecimal notation) see Kaggle. * @param month see Kaggle. * @param reportedBy see Kaggle. * @param fallsWithin see Kaggle. - * @param longitude (optional Double) the longitude of the incident. - * @param latitude (optional Double) the latitude of the incident. - * @param location see Kaggle. - * @param lsoaCode see Kaggle. - * @param lsoaName see Kaggle. + * @param maybeLocation a CrimeLocation. * @param crimeType see Kaggle. * @param lastOutcomeCategory see Kaggle. * @param context see Kaggle. */ -case class Crime(crimeID: Option[BigInt], +case class Crime(sequence: Sequence, + maybeCrimeId: Option[BigInt], month: String, reportedBy: String, fallsWithin: String, - longitude: Option[Double], - latitude: Option[Double], - location: String, - lsoaCode: String, - lsoaName: String, + maybeLocation: Option[CrimeLocation], crimeType: String, lastOutcomeCategory: String, - context: String) { - def brief: Option[CrimeLocation] = for (long <- longitude; lat <- latitude) yield CrimeLocation(crimeID, long, lat) -} + context: String) extends Sequential { + def isValid: Boolean = maybeCrimeId.isDefined && maybeLocation.exists(_.isValid) -case class CrimeLocation(crimeID: Option[BigInt], - longitude: Double, - latitude: Double) { + def brief: Option[CrimeBrief] = for (crimeId <- maybeCrimeId; location <- maybeLocation) yield CrimeBrief(crimeId, location.longitude, location.latitude) } -object CrimeParser extends CellParsers { +/** + * Companion object to Crime. + */ +object Crime extends CellParsers with CsvRenderers { - /** - * Precede each upper case letter (or digit) with _. - */ - def camelToSnakeCaseColumnNameMapper(w: String): String = w.replaceAll("([A-Z\\d])", " $1") + import CsvRenderers.{CsvRendererDouble, CsvRendererString} + import com.phasmidsoftware.render.CsvGenerators._ + + val filename: String = "2023-01-metropolitan-street.csv" + val ioResourceNotAvailableOnGithub: IO[URL] = ioResource[Crime](filename) + + // TODO merge the two copies of this sample file into one (it needs to be at the root level of resources) + val sampleFile = "2023-01-metropolitan-street-sample.csv" + val ioSampleResource: IO[URL] = ioResource[Crime](sampleFile) + + implicit object crimeValidity extends Validity[Crime] { + def isValid(c: Crime): Boolean = c.isValid + } implicit object BigIntCellParser extends SingleCellParser[BigInt] { def convertString(w: String): Try[BigInt] = implicitly[Parseable[BigInt]].parse(w, Some("16")) } - implicit val movieColumnHelper: ColumnHelper[Crime] = columnHelper(camelToSnakeCaseColumnNameMapper _, - "crimeID" -> "Crime ID") - - implicit val crimeIdParser: CellParser[Option[BigInt]] = cellParserOption[BigInt] - implicit val movieParser: CellParser[Crime] = cellParser12(Crime.apply) + implicit val crimeColumnHelper: ColumnHelper[Crime] = columnHelper(camelToSnakeCaseColumnNameMapper _, + "maybeCrimeId" -> "Crime ID") implicit object CrimeConfig extends DefaultRowConfig { override val listEnclosure: String = "" } + implicit val crimeIdParser: CellParser[Option[BigInt]] = cellParserOption[BigInt] + implicit val crimeLocationParser: CellParser[Option[CrimeLocation]] = cellParserOption[CrimeLocation] + implicit val crimeParser: CellParser[Crime] = cellParser9(Crime.apply) implicit val parser: StandardRowParser[Crime] = StandardRowParser.create[Crime] + implicit val crimeOrdering: Ordering[Crime] = Sequential.ordering[Crime] - trait CrimeTableParser extends StringTableParser[Table[Crime]] { - type Row = Crime + implicit val crimeIDRenderer: CsvRenderer[BigInt] = new CsvRenderer[BigInt] { + val csvAttributes: CsvAttributes = implicitly[CsvAttributes] - val maybeFixedHeader: Option[Header] = None + def render(t: BigInt, attrs: Map[String, String]): String = t.toString(16) + } - val headerRowsToRead: Int = 1 + private val generators = new CsvGenerators {} + implicit val crimeIdRenderer: CsvRenderer[Option[BigInt]] = optionRenderer[BigInt]() + implicit val crimeIdGenerator: CsvGenerator[Option[BigInt]] = generators.optionGenerator[BigInt] + implicit val geoRenderer: CsvRenderer[Option[Double]] = optionRenderer[Double]() + implicit val geoGenerator: CsvGenerator[Option[Double]] = generators.optionGenerator[Double] + implicit val crimeLocationProduct: CsvProduct[Option[CrimeLocation]] = optionProduct[CrimeLocation]() + implicit val crimeRenderer: CsvProduct[Crime] = rendererGenerator9(Crime.apply) + // CONSIDER why doesn't importing the implicit object CrimeParser.CrimeTableParser work? + implicit val p: CrimeParser.CrimeTableParser = new CrimeParser.CrimeTableParser(true, _ => true) + + import cats.effect.IO + + def doMain(ioResource: IO[URL])(implicit random: Random): IO[String] = + for { + url <- ioResource // get the URL for either the complete file or a sample file. + ct <- IOUsing(IO(Source.fromURL(url)))(x => Table.parseSource(x)) // open/close resource and parse it as a Table[Crime]. + lt <- IO(ct.filterValid.mapOptional(m => m.brief)) // filter according to validity and then convert rows to CrimeBrief. + st <- IO(lt.sample(120)) // sample 1 in every (approximately) 120 rows. + w <- st.toCSV // write the table out in CSV format. + } yield w +} - override val forgiving: Boolean = true +/** + * CrimeLocation. + * + * @param longitude the longitude of the incident. + * @param latitude the latitude of the incident. + * @param location see Kaggle. + * @param lsoaCode see Kaggle. + * @param lsoaName see Kaggle. + */ +case class CrimeLocation(longitude: Double, + latitude: Double, + location: String, + lsoaCode: String, + lsoaName: String + ) { + def isValid: Boolean = CrimeLocation.isValid(longitude, latitude, lsoaCode) +} - val rowParser: RowParser[Row, String] = implicitly[RowParser[Row, String]] +/** + * Companion object to CrimeLocation. + */ +object CrimeLocation extends CellParsers with CsvRenderers { + private val invalidLSOACodes = Seq("E01032496", "E01011349", "E01024436", "E01032969", "E01021416", "E01021427", "E01016619", "E01015693", "E01032731", "E01030261", "E01023724", "E01023548", "E01009385", "E01016920", "E01000387", "E01026188", "E01030384", "E01017765", "E01031789", "E01003802", "E01016215", "E01010676", "E01024821", "E01000755", "E01000686", "E01027148", "E01033022", "E01028101", "E01024261", "E01016608", "E01030606", "E01016464", "E01023805", "E01009923", "E01033451", "E01001126", "E01030300", "E01021765", "E01010326", "E01024172", "E01015772", "E01021945", "E01000833", "E01010054", "E01031587", "E01005692", "E01023302", "E01010635", "E01002255", "E01030333", "E01024475", "E01033212", "E01016006", "E01002922", "E01006386", "E01032645", "E01033739", "E01015982", "E01030668", "E01016540", "E01018996", "E01021818", "E01024429", "E01002288", "E01016074", "E01002462", "E01003466", "E01023951", "E01020995", "E01030350", "E01015935", "E01023344", "E01024243", "E01017810", "E01017392", "E01003846", "E01030851", "E01033542", "E01015992", "E01023793", "E01023840", "E01030548", "E01004707", "E01024247", "E01003008", "E01001107", "E01032979", "E01016129", "E01023963", "E01023778", "E01024189", "E01031333", "E01030685", "E01005197", "E01032799", "E01021749", "E01000345", "E01023580", "E01030306", "E01023850", "E01030743", "E01002359", "E01023849", "E01030751", "E01008709", "E01006832", "E01024155", "E01023861", "E01023908", "E01023644", "E01024185", "E01002995", "E01017811", "E01030323", "E01023341", "E01023649", "E01030704", "E01030856", "E01025277", "E01021954", "E01025627", "E01032684", "E01000356", "E01006194", "E01022295", "E01032571", "E01013916", "E01023573", "E01030392", "E01024152", "E01003138", "E01005568", "E01024149", "E01004338", "E01017619", "E01023942", "E01021310", "W01000010", "E01023378", "E01015688", "E01000425", "E01021663", "E01023444", "E01032378", "E01030933", "E01024047", "E01017989", "E01017423", "E01011036", "E01010425", "E01030201", "E01025767", "E01030735", "E01021436", "E01021447", "E01015777", "E01027711", "E01000717", "E01030610", "E01000436", "E01000836", "E01021806", "E01000371", "E01030855", "E01023352", "E01026959", "E01020971", "E01021319", "E01004097", "E01015734", "E01028660", "E01009709", "E01015241", "E01001058", "E01024162", "E01000461", "E01024745", "E01013665", "E01016474", "E01010813", "E01026591", "E01030566", "E01024186", "E01012454", "E01029475", "E01015782", "E01014706", "E01000003", "E01030531", "E01016011", "E01023541", "E01024783", "E01016098", "E01023844", "E01001069", "E01031819", "E01016939", "E01024169", "E01033747", "W01001867", "W01000733", "E01021469", "E01023877", "E01003457", "E01020507", "E01016912", "E01030717", "E01028843", "E01004163", "E01021324", "E01026868", "E01024420", "E01024158", "E01016482", "E01021500", "E01023758", "E01033742", "E01000949", "E01020086", "E01015808", "E01024136", "E01000932", "E01016034", "E01017155", "E01001456", "E01023842", "E01016549", "E01002388", "E01008551", "E01030344", "E01005798", "E01028331", "E01017812", "E01023339", "E01030310", "E01002155", "E01023899", "E01017139", "E01033135", "E01025802", "E01002699", "E01006211", "E01016602", "E01015773", "E01018219", "E01033164", "E01003676", "E01030853", "E01033345", "E01015902", "E01016247", "E01004475", "E01015951", "E01003691", "E01001350", "E01015795", "E01006633", "E01023559", "E01027320", "E01014073", "E01016385", "E01016450", "E01030755", "E01000723", "E01030744", "E01013258", "E01023913", "E01024391", "E01031723", "E01001236", "E01011992") - protected def builder(rows: Iterable[Crime], header: Header): Table[Row] = HeadedTable(Content(rows), header) - } + def isValid(longitude: Double, latitude: Double, lsoaCode: String): Boolean = + !(latitude > 51.7 || longitude > 0.3 || latitude < 51.2 || longitude < -0.51 || invalidLSOACodes.contains(lsoaCode)) - implicit object CrimeTableParser extends CrimeTableParser -} + /** + * Precede each upper case letter (or digit) with _. + */ + def camelToSnakeCaseColumnNameMapper(w: String): String = w.replaceAll("([A-Z\\d])", " $1") -object CrimeRenderer extends CsvRenderers { + implicit val locationColumnHelper: ColumnHelper[CrimeLocation] = columnHelper(camelToSnakeCaseColumnNameMapper _, + "lsoaCode" -> "LSOA code", + "lsoaName" -> "LSOA name" + ) - import CsvRenderers._ - import com.phasmidsoftware.render.CsvGenerators._ + implicit val locationParser: CellParser[CrimeLocation] = cellParser5(CrimeLocation.apply) - private val generators = new CsvGenerators {} + import CsvGenerators._ + import CsvRenderers._ - implicit val bigIntRenderer: CsvRenderer[BigInt] = new CsvRenderer[BigInt] { - val csvAttributes: CsvAttributes = implicitly[CsvAttributes] + implicit val locationRenderer: CsvProduct[CrimeLocation] = rendererGenerator5(CrimeLocation.apply) +} - def render(t: BigInt, attrs: Map[String, String]): String = t.toString(16) - } - implicit val crimeIdRenderer: CsvRenderer[Option[BigInt]] = optionRenderer[BigInt]() - implicit val crimeIdGenerator: CsvGenerator[Option[BigInt]] = generators.optionGenerator[BigInt] - implicit val geoRenderer: CsvRenderer[Option[Double]] = optionRenderer[Double]() - implicit val geoGenerator: CsvGenerator[Option[Double]] = generators.optionGenerator[Double] - implicit val crimeRenderer: CsvProduct[Crime] = rendererGenerator12(Crime.apply) +/** + * Case class to model a very brief version of a crime (only the ID, and geographic location). + * This is for the INFO6205 project, Spring 2023, solving the TSP. + * + * @param crimeID (BigInt) the crime ID of the incident, expressed in CSV in hexadecimal. + * @param longitude (Double) the longitude of the incident. + * @param latitude (Double) the latitude of the incident. + */ +case class CrimeBrief(crimeID: BigInt, + longitude: Double, + latitude: Double) { } -object CrimeLocationRenderer extends CsvRenderers { +object CrimeBrief extends CsvRenderers { + implicit val crimeBriefOrdering: Ordering[CrimeBrief] = NonSequential.ordering[CrimeBrief, BigInt](c => c.crimeID) - import CsvRenderers._ + import Crime.crimeIDRenderer + import CsvRenderers.CsvRendererDouble import com.phasmidsoftware.render.CsvGenerators._ private val generators = new CsvGenerators {} + implicit val crimeIdRenderer: CsvRenderer[Option[BigInt]] = optionRenderer[BigInt]("unidentified") + implicit val crimeIdGenerator: CsvGenerator[Option[BigInt]] = generators.optionGenerator + implicit val crimeRenderer: CsvProduct[CrimeBrief] = rendererGenerator3(CrimeBrief.apply) +} - implicit val bigIntRenderer: CsvRenderer[BigInt] = new CsvRenderer[BigInt] { - val csvAttributes: CsvAttributes = implicitly[CsvAttributes] +object CrimeParser extends CellParsers { + case class CrimeTableParser(override val forgiving: Boolean, override val predicate: Try[Crime] => Boolean) extends StringTableParser[Table[Crime]] with SelectiveParser[Crime, Table[Crime]] { + type Row = Crime - def render(t: BigInt, attrs: Map[String, String]): String = t.toString(16) + val maybeFixedHeader: Option[Header] = None + + val headerRowsToRead: Int = 1 + + val rowParser: RowParser[Row, String] = implicitly[RowParser[Row, String]] + + def setForgiving(b: Boolean): TableParser[Table[Crime]] = copy(forgiving = b) + + def setPredicate(p: Try[Crime] => Boolean): TableParser[Table[Crime]] = copy(predicate = p) + + protected def builder(rows: Iterable[Crime], header: Header): Table[Row] = HeadedTable(Content(rows), header) } - implicit val crimeIdRenderer: CsvRenderer[Option[BigInt]] = optionRenderer[BigInt]("unidentified") - implicit val crimeIdGenerator: CsvGenerator[Option[BigInt]] = generators.optionGenerator - implicit val crimeRenderer: CsvProduct[CrimeLocation] = rendererGenerator3(CrimeLocation.apply) + + implicit object CrimeTableParser extends CrimeTableParser(true, _ => true) } +/** + * Main program to create a sample of valid rows from the complete Metropolitan crime dataset. + */ +object Main extends App { + + import cats.effect.IO + + implicit val random: Random = new Random() + // NOTE: we specify the complete Metropolitan file (not available on GitHub). + val wi: IO[String] = Crime.doMain(Crime.ioResourceNotAvailableOnGithub) + + println(EvaluateIO(wi, Timeout(Span(10, Seconds)))) +} \ No newline at end of file diff --git a/src/test/scala/com/phasmidsoftware/examples/crime/CrimeSpec.scala b/src/test/scala/com/phasmidsoftware/examples/crime/CrimeSpec.scala index 086bf026..c542e9ef 100644 --- a/src/test/scala/com/phasmidsoftware/examples/crime/CrimeSpec.scala +++ b/src/test/scala/com/phasmidsoftware/examples/crime/CrimeSpec.scala @@ -1,33 +1,45 @@ package com.phasmidsoftware.examples.crime -import cats.effect.IO -import com.phasmidsoftware.parse.{RawTableParser, TableParser} -import com.phasmidsoftware.table.{Analysis, HeadedTable, RawTable, Table} +import cats.effect.{IO, Resource} +import com.phasmidsoftware.parse.{RawTableParser, StandardStringsParser, TableParser} +import com.phasmidsoftware.table._ import com.phasmidsoftware.util.EvaluateIO.matchIO import com.phasmidsoftware.util.FP.resource import com.phasmidsoftware.util.{FP, IOUsing} +import java.io.FileWriter +import java.net.URL import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Seconds, Span} import scala.io.Source -import scala.util.Random +import scala.util.{Random, Success, Try} class CrimeSpec extends AnyFlatSpec with Matchers { + behavior of "CrimeLocation" + + it should "parse from Strings" in { + val header: Header = Header.create("longitude", "latitude", "location", "LSOA code", "LSOA name") + val parser = StandardStringsParser[CrimeLocation]() + val location: Try[CrimeLocation] = parser.parse((Seq("0.140127", "51.588913", "On or near Beansland Grove", "E01000027", "Barking and Dagenham 001A"), 0))(header) + location shouldBe Success(CrimeLocation(0.140127, 51.588913, "On or near Beansland Grove", "E01000027", "Barking and Dagenham 001A")) + } + behavior of "Crime" - val crimeFile = "2023-01-metropolitan-street-sample.csv" + val sampleFile = "2023-01-metropolitan-street-sample.csv" + val triedCrimeSampleResource: Try[URL] = resource[CrimeSpec](sampleFile) it should "be ingested and analyzed as a RawTable" in { // Set up the source - val sy: IO[Source] = IO.fromTry(for (u <- resource[CrimeSpec](crimeFile)) yield Source.fromURL(u)) + val si: IO[Source] = IO.fromTry(for (u <- triedCrimeSampleResource) yield Source.fromURL(u)) // Set up the parser (we set the predicate only for demonstration purposes) val parser: RawTableParser = RawTableParser().setPredicate(TableParser.sampler(10)) // Create the table - val wsty: IO[RawTable] = parser.parse(sy) + val wsty: IO[RawTable] = parser.parse(si) matchIO(wsty, Timeout(Span(10, Seconds))) { case t@HeadedTable(r, _) => @@ -39,15 +51,29 @@ class CrimeSpec extends AnyFlatSpec with Matchers { } } - // FIXME this is because the output is essentially in random order. - ignore should "be ingested and written out in brief to CSV" in { - import CrimeLocationRenderer._ + it should "get the order right for Crime" in { + val sequence1 = Sequence(1) + val sequence2 = sequence1.next + val x1 = Crime(sequence1, None, "", "", "", None, "", "", "") + val x2 = Crime(sequence2, None, "", "", "", None, "", "", "") + val co = implicitly[Ordering[Crime]] + co.compare(x1, x2) shouldBe -1 + } + + it should "get the order right for CrimeBrief" in { + val x1 = CrimeBrief(BigInt(0), 0.0, 0.0) + val x2 = CrimeBrief(BigInt(1), 0.0, 0.0) + val co = implicitly[Ordering[CrimeBrief]] + co.compare(x1, x2) shouldBe -1 + co.compare(x2, x1) shouldBe 1 + } + + it should "be ingested and written out in brief to CSV" in { import CrimeParser._ implicit val random: Random = new Random(0) - val cti: IO[Table[Crime]] = IOUsing(Source.fromURL(classOf[Crime].getResource(crimeFile)))(x => Table.parseSource(x)) - val wi: IO[String] = for { - ct <- cti + url <- Crime.ioSampleResource + ct <- IOUsing(Source.fromURL(url))(x => Table.parseSource(x)) lt <- IO(ct.mapOptional(m => m.brief)) st <- IO(lt.filter(FP.sampler(10))) w <- st.toCSV @@ -55,11 +81,34 @@ class CrimeSpec extends AnyFlatSpec with Matchers { matchIO(wi, Timeout(Span(20, Seconds))) { case w => - // NOTE that the output from a parallel store is random. This may not always work. - w should startWith( - """crimeID,longitude,latitude - |8536e93fb3ce916daa4251bd53c1a4416ba4159a938340be4a7c40cd4873bfcf,-0.681541,50.792113""".stripMargin) + // NOTE that the output from a parallel store is random (why?). + w should startWith("""crimeID,longitude,latitude""".stripMargin) } } + it should "doMain" in { + implicit val random: Random = new Random(0) + matchIO(Crime.doMain(Crime.ioSampleResource), Timeout(Span(20, Seconds))) { + case w => w.lines().count() shouldBe 18 + } + } + + it should "use Resource" in { + import CrimeParser._ + import cats.effect.unsafe.implicits.global + implicit val random: Random = new Random(0) + val filename = "tmp/Crime.use.Resource.csv" + val writeResource = Resource.make(IO(new FileWriter(filename)))(fw => IO(fw.close())) + val wi: IO[Unit] = for { + url <- Crime.ioSampleResource + readResource = Resource.make(IO(Source.fromURL(url)))(src => IO(src.close())) + ct <- readResource.use(src => Table.parseSource(src)) + lt <- IO(ct.mapOptional(m => m.brief)) + st <- IO(lt.filter(FP.sampler(10))) + w <- st.toCSV + _ <- writeResource.use(fw => IO(fw.write(w))) + } yield () + + wi.unsafeRunSync() + } } diff --git a/src/test/scala/com/phasmidsoftware/parse/CellParserSpec.scala b/src/test/scala/com/phasmidsoftware/parse/CellParserSpec.scala index 35adceba..d1061bf1 100644 --- a/src/test/scala/com/phasmidsoftware/parse/CellParserSpec.scala +++ b/src/test/scala/com/phasmidsoftware/parse/CellParserSpec.scala @@ -121,6 +121,8 @@ class CellParserSpec extends flatspec.AnyFlatSpec with should.Matchers { p.convertString("test") should matchPattern { case Failure(_) => } } + // CONSIDER moving this into the it directory. + // TODO why does this take so long? it should "parse option URL" in { val p = implicitly[CellParser[Option[URL]]] diff --git a/src/test/scala/com/phasmidsoftware/render/CsvGeneratorSpec.scala b/src/test/scala/com/phasmidsoftware/render/CsvGeneratorSpec.scala new file mode 100644 index 00000000..7686bc46 --- /dev/null +++ b/src/test/scala/com/phasmidsoftware/render/CsvGeneratorSpec.scala @@ -0,0 +1,23 @@ +package com.phasmidsoftware.render + +import com.phasmidsoftware.examples.crime.Crime.crimeIdGenerator +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should + +class CsvGeneratorSpec extends AnyFlatSpec with should.Matchers { + + behavior of "CsvGenerator" + + it should "toColumnName BigInt" in { + import CsvGenerators._ + val csvGenerator: CsvGenerator[BigInt] = implicitly[CsvGenerator[BigInt]] + val header = csvGenerator.toColumnName(None, "id") + header shouldBe "id" + } + it should "toColumnName Option[BigInt]" in { + val csvGenerator: CsvGenerator[Option[BigInt]] = implicitly[CsvGenerator[Option[BigInt]]] + val header = csvGenerator.toColumnName(None, "maybeId") + header shouldBe "id" + } + +} diff --git a/src/test/scala/com/phasmidsoftware/render/CsvRenderersSpec.scala b/src/test/scala/com/phasmidsoftware/render/CsvRenderersSpec.scala index 64b1a3c6..11ad6095 100644 --- a/src/test/scala/com/phasmidsoftware/render/CsvRenderersSpec.scala +++ b/src/test/scala/com/phasmidsoftware/render/CsvRenderersSpec.scala @@ -24,6 +24,8 @@ class CsvRenderersSpec extends AnyFlatSpec with should.Matchers { object IntPair { + implicit val intPairOrdering: Ordering[IntPair] = NonSequential.ordering[IntPair, Int](c => c.a) + object IntPairParser extends JavaTokenParsers { lazy val pair: Parser[(Int, Int)] = wholeNumber ~ wholeNumber ^^ { case x ~ y => (x.toInt, y.toInt) } } @@ -305,6 +307,8 @@ class CsvRenderersSpec extends AnyFlatSpec with should.Matchers { object DailyRaptorReport { + implicit val dailyRaptorReportOrdering: Ordering[DailyRaptorReport] = NonSequential.ordering[DailyRaptorReport, LocalDate](c => c.date) + object DailyRaptorReportParser extends CellParsers { private val raptorReportDateFormatter = DateTimeFormat.forPattern("MM/dd/yyyy") @@ -391,6 +395,7 @@ class CsvRenderersSpec extends AnyFlatSpec with should.Matchers { case class NestedRaptorReport(date: LocalDate, weatherHawks: WeatherHawks) object NestedRaptorReport { + implicit val nestedRaptorReportOrdering: Ordering[NestedRaptorReport] = NonSequential.ordering[NestedRaptorReport, LocalDate](c => c.date) object NestedRaptorReportParser extends CellParsers { private val raptorReportDateFormatter = DateTimeFormat.forPattern("MM/dd/yyyy") diff --git a/src/test/scala/com/phasmidsoftware/table/AnalysisSpec.scala b/src/test/scala/com/phasmidsoftware/table/AnalysisSpec.scala index ede352aa..64ef7f4b 100644 --- a/src/test/scala/com/phasmidsoftware/table/AnalysisSpec.scala +++ b/src/test/scala/com/phasmidsoftware/table/AnalysisSpec.scala @@ -32,8 +32,8 @@ class AnalysisSpec extends AnyFlatSpec with Matchers { analysis.columns shouldBe 87 analysis.columnMap.size shouldBe 87 analysis.columnMap("bedrooms") should matchPattern { case Column("Int", false, _) => } - analysis.columnMap("accommodates").toString should startWith("Int (range: 1.0-10.0, mean: 2.783464566929134, stdDev: 1.7670324685210") - analysis.columnMap("license").toString shouldBe "optional Int" + analysis.columnMap("accommodates").toString should startWith("Int: total: 254\n (range: 1.0-10.0, mean: 2.783464566929134, stdDev: 1.7670324685210") + analysis.columnMap("license").toString shouldBe "optional Int: " } } @@ -47,10 +47,14 @@ class AnalysisSpec extends AnyFlatSpec with Matchers { val maybeColumn: Option[Column] = sequence(z) flatMap (ws => make(ws)) maybeColumn should matchPattern { case Some(Column(_, _, Some(_))) => } maybeColumn.get match { - case Column("Int", false, Some(statistics)) => + case Column("Int", false, Some(statistics: Statistics)) => statistics.mu shouldBe 2.783464566929134 case x => fail(x.toString) } } } + + behavior of "Histogram" + + it should "make a histogram" } diff --git a/src/test/scala/com/phasmidsoftware/table/ContentSpec.scala b/src/test/scala/com/phasmidsoftware/table/ContentSpec.scala index a5037499..7dacb17c 100644 --- a/src/test/scala/com/phasmidsoftware/table/ContentSpec.scala +++ b/src/test/scala/com/phasmidsoftware/table/ContentSpec.scala @@ -4,12 +4,39 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should import scala.collection.parallel.CollectionConverters._ +/** + * ContentSpec + * + * NOTE: The methods here depend on the behavior of the ParIterable parameter of Content. + */ class ContentSpec extends AnyFlatSpec with should.Matchers { behavior of "Content" + it should "ordered" in { + val target: Content[Int] = Content(List(1, 2, 3)) + target.ordered shouldBe List(1, 2, 3) + } + + it should "sorted" in { + val target: Content[Int] = Content(List(2, 1, 3)) + target.sorted shouldBe Content(List(1, 2, 3)) + } + it should "drop" in { + val target: Content[Int] = Content(List(1, 2, 3)) + // CONSIDER forcing sorted on the drop method. + target.drop(1).toSeq shouldBe List(2, 3) + } + + it should "take" in { + val target: Content[Int] = Content(List(1, 2, 3)) + target.take(2).toSeq shouldBe List(1, 2) + } + it should "slice" in { + val target: Content[Int] = Content(List(1, 2, 3, 4)) + target.slice(1, 3).toSeq shouldBe List(2, 3) } it should "toIndexedSeq" in { @@ -44,10 +71,6 @@ class ContentSpec extends AnyFlatSpec with should.Matchers { } - it should "slice" in { - - } - it should "toArray" in { } @@ -72,10 +95,6 @@ class ContentSpec extends AnyFlatSpec with should.Matchers { } - it should "take" in { - - } - it should "toSeq" in { } diff --git a/src/test/scala/com/phasmidsoftware/table/Movie.scala b/src/test/scala/com/phasmidsoftware/table/Movie.scala index 5c3d9e04..22356415 100644 --- a/src/test/scala/com/phasmidsoftware/table/Movie.scala +++ b/src/test/scala/com/phasmidsoftware/table/Movie.scala @@ -14,6 +14,8 @@ import scala.util.Try * * Created by scalaprof on 9/12/16. * + * CONSIDER moving this into examples package + * * Common questions in this assignment: * 1. Where is main method? * In most case, you don't need to run main method for assignments. @@ -219,6 +221,8 @@ object Movie { implicit val generatorAttributeSet: CsvGenerator[AttributeSet] = csvGenerators.generator1(fAttributeSet) csvGenerators.generator11(Movie.apply) } + + implicit val orderingTeamProject: Ordering[Movie] = NonSequential.ordering[Movie, String](p => p.title) } // CONSIDER removing the csvAttributes parameter and making it an object. diff --git a/src/test/scala/com/phasmidsoftware/table/SequentialSpec.scala b/src/test/scala/com/phasmidsoftware/table/SequentialSpec.scala new file mode 100644 index 00000000..44da25b7 --- /dev/null +++ b/src/test/scala/com/phasmidsoftware/table/SequentialSpec.scala @@ -0,0 +1,94 @@ +package com.phasmidsoftware.table + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should +import scala.util.{Failure, Success, Try} + +class SequentialSpec extends AnyFlatSpec with should.Matchers { + + case class Tester(x: Int) + + object Tester { + val negOne: Tester = Tester(-1) + val zero: Tester = Tester(0) + val one: Tester = Tester(1) + } + + case class TesterOpt(x: Option[Int]) + + object TesterOpt { + val negOne: TesterOpt = TesterOpt(Some(-1)) + val zero: TesterOpt = TesterOpt(Some(0)) + val one: TesterOpt = TesterOpt(Some(1)) + val none: TesterOpt = TesterOpt(None) + } + + case class TesterTry(x: Try[Int]) + + object TesterTry { + val negOne: TesterTry = TesterTry(Success(-1)) + val zero: TesterTry = TesterTry(Success(0)) + val one: TesterTry = TesterTry(Success(1)) + val failure: TesterTry = TesterTry(Failure(new NoSuchElementException)) + } + + case class TesterStringOpt(x: Option[String]) + + object TesterStringOpt { + val a: TesterStringOpt = TesterStringOpt(Some("a")) + val b: TesterStringOpt = TesterStringOpt(Some("b")) + val c: TesterStringOpt = TesterStringOpt(Some("c")) + val none: TesterStringOpt = TesterStringOpt(None) + } + + behavior of "NonSequential" + + it should "ordering" in { + val ordering = NonSequential.ordering[Tester, Int](_.x) + ordering.compare(Tester.zero, Tester.one) shouldBe -1 + ordering.compare(Tester.zero, Tester.zero) shouldBe 0 + ordering.compare(Tester.one, Tester.zero) shouldBe 1 + ordering.compare(Tester.negOne, Tester.zero) shouldBe -1 + ordering.compare(Tester.one, Tester.one) shouldBe 0 + ordering.compare(Tester.one, Tester.negOne) shouldBe 1 + } + + it should "optionalOrdering" in { + val ordering = NonSequential.optionalOrdering[TesterOpt, Int](_.x) + ordering.compare(TesterOpt.zero, TesterOpt.one) shouldBe -1 + ordering.compare(TesterOpt.zero, TesterOpt.zero) shouldBe 0 + ordering.compare(TesterOpt.one, TesterOpt.zero) shouldBe 1 + ordering.compare(TesterOpt.negOne, TesterOpt.zero) shouldBe -1 + ordering.compare(TesterOpt.one, TesterOpt.one) shouldBe 0 + ordering.compare(TesterOpt.one, TesterOpt.negOne) shouldBe 1 + ordering.compare(TesterOpt.none, TesterOpt.one) shouldBe -1 + ordering.compare(TesterOpt.none, TesterOpt.none) shouldBe 0 + ordering.compare(TesterOpt.negOne, TesterOpt.none) shouldBe -1 + } + + it should "optionalStringOrdering" in { + val ordering = NonSequential.optionalOrdering[TesterStringOpt, String](_.x) + ordering.compare(TesterStringOpt.b, TesterStringOpt.c) shouldBe -1 + ordering.compare(TesterStringOpt.b, TesterStringOpt.b) shouldBe 0 + ordering.compare(TesterStringOpt.c, TesterStringOpt.b) shouldBe 1 + ordering.compare(TesterStringOpt.a, TesterStringOpt.b) shouldBe -1 + ordering.compare(TesterStringOpt.c, TesterStringOpt.c) shouldBe 0 + ordering.compare(TesterStringOpt.c, TesterStringOpt.a) shouldBe 2 + ordering.compare(TesterStringOpt.none, TesterStringOpt.c) shouldBe -1 + ordering.compare(TesterStringOpt.none, TesterStringOpt.none) shouldBe 0 + ordering.compare(TesterStringOpt.a, TesterStringOpt.none) shouldBe 1 + } + + it should "tryOrdering" in { + val ordering = NonSequential.tryOrdering[TesterTry, Int](_.x) + ordering.compare(TesterTry.zero, TesterTry.one) shouldBe -1 + ordering.compare(TesterTry.zero, TesterTry.zero) shouldBe 0 + ordering.compare(TesterTry.one, TesterTry.zero) shouldBe 1 + ordering.compare(TesterTry.negOne, TesterTry.zero) shouldBe -1 + ordering.compare(TesterTry.one, TesterTry.one) shouldBe 0 + ordering.compare(TesterTry.one, TesterTry.negOne) shouldBe 1 + ordering.compare(TesterTry.failure, TesterTry.one) shouldBe -1 + ordering.compare(TesterTry.failure, TesterTry.failure) shouldBe 0 + ordering.compare(TesterTry.negOne, TesterTry.failure) shouldBe -1 + } +} diff --git a/src/test/scala/com/phasmidsoftware/table/TableSpec.scala b/src/test/scala/com/phasmidsoftware/table/TableSpec.scala index 0cf4531b..d559a98f 100644 --- a/src/test/scala/com/phasmidsoftware/table/TableSpec.scala +++ b/src/test/scala/com/phasmidsoftware/table/TableSpec.scala @@ -5,13 +5,13 @@ package com.phasmidsoftware.table import cats.effect.IO -import cats.implicits.catsSyntaxParallelAp +import cats.implicits.catsSyntaxNonEmptyParallelAp import com.phasmidsoftware.parse._ import com.phasmidsoftware.render._ import com.phasmidsoftware.table.Table.parseResource import com.phasmidsoftware.util.EvaluateIO.matchIO import com.phasmidsoftware.util.{EvaluateIO, TryUsing} -import com.phasmidsoftware.write.{Node, TreeWriter, Writable} +import com.phasmidsoftware.write.{Node, TreeWriter, Writable, WritableSpec} import java.io.{File, FileWriter, InputStream} import java.net.URL import org.scalatest.concurrent.PatienceConfiguration.Timeout @@ -107,8 +107,9 @@ class TableSpec extends flatspec.AnyFlatSpec with should.Matchers { } // NOTE: this test can be flaky. Perhaps we should just use zip instead of parProduct. + // TODO we are relying on the existence of WritableSpec.complexFile, which may not exist. it should "parse table from raw file" in { - val z1: IO[Table[RawRow]] = Table.parseFileRaw(new File("output.csv"), TableParser.includeAll, Some(Header(Seq(Seq("a", "b"))))) + val z1: IO[Table[RawRow]] = Table.parseFileRaw(new File(WritableSpec.complexFile), TableParser.includeAll, Some(Header(Seq(Seq("a", "b"))))) val z2: IO[Table[RawRow]] = Table.parseFileRaw("src/test/resources/com/phasmidsoftware/table/intPairs.csv", TableParser.includeAll) matchIO(z1 parProduct z2) { case (a@HeadedTable(_, _), b@HeadedTable(_, _)) => @@ -116,19 +117,21 @@ class TableSpec extends flatspec.AnyFlatSpec with should.Matchers { } } - it should "write table to the file" in { + it should "write table to file" in { val hdr = Header(Seq(Seq("a", "b"))) val row1 = Row(Seq("1", "2"), hdr, 1) val table = Table(Seq(row1), Some(hdr)) - val resultIO = for {_ <- Table.writeCSVFileRow(table, new File("output.csv")) - _ = println(s"written to file output.csv") - y <- Table.parseFileRaw("output.csv", TableParser.includeAll) + implicit val z: Ordering[Row] = Content.noOrdering[Row] + val outputFile = "tmp/Table-write Table To File.csv" + val resultIO = for {_ <- Table.writeCSVFileRow(table, new File(outputFile)) + _ <- IO.println(s"written to file " + outputFile) + y <- Table.parseFileRaw(outputFile, TableParser.includeAll) } yield y matchIO(resultIO) { case xt@HeadedTable(_, _) => xt.content.head.toString() shouldBe """A="1", B="2"""" } val tableWithoutHead = Table(Seq(row1), None) - the[TableException] thrownBy Table.writeCSVFileRow(tableWithoutHead, new File("output.csv")) + the[TableException] thrownBy Table.writeCSVFileRow(tableWithoutHead, new File(outputFile)) } it should "parse from Iterator[String]" in { @@ -263,7 +266,7 @@ class TableSpec extends flatspec.AnyFlatSpec with should.Matchers { } } - behavior of "other" + behavior of "Other" it should "do iterator" in { import IntPair._ @@ -314,51 +317,43 @@ class TableSpec extends flatspec.AnyFlatSpec with should.Matchers { } } - it should "drop" in { + it should "empty" in { import IntPair._ matchIO(Table.parse(Seq("1 2", "42 99"))) { case xt@HeadedTable(_, _) => - xt.drop(1).content.toSeq shouldBe Seq(IntPair(42, 99)) + xt.empty.content.toSeq shouldBe Seq.empty } } -// it should "dropRight" in { -// import IntPair._ -// matchIO(Table.parse(Seq("1 2", "42 99"))) { -// case xt@HeadedTable(_, _) => -// xt.dropRight(1).rows shouldBe Seq(IntPair(1, 2)) -// } -// } - - it should "empty" in { + it should "filter" in { import IntPair._ - matchIO(Table.parse(Seq("1 2", "42 99"))) { + matchIO(Table.parse(Seq("3 4", "1 2", "42 99"))) { case xt@HeadedTable(_, _) => - xt.empty.content.toSeq shouldBe Seq.empty + xt.filter(_.equals(IntPair(3, 4))).content.toSeq shouldBe Seq(IntPair(3, 4)) } } - it should "dropWhile" in { + it should "filterNot" in { import IntPair._ matchIO(Table.parse(Seq("3 4", "1 2", "42 99"))) { case xt@HeadedTable(_, _) => - xt.dropWhile(_.equals(IntPair(3, 4))).content.toSeq shouldBe Seq(IntPair(1, 2), IntPair(42, 99)) + xt.filterNot(_.equals(IntPair(3, 4))).content.toSeq shouldBe Seq(IntPair(1, 2), IntPair(42, 99)) } } - it should "filter" in { + it should "drop" in { import IntPair._ - matchIO(Table.parse(Seq("3 4", "1 2", "42 99"))) { + matchIO(Table.parse(Seq("1 2", "42 99"))) { case xt@HeadedTable(_, _) => - xt.filter(_.equals(IntPair(3, 4))).content.toSeq shouldBe Seq(IntPair(3, 4)) + xt.drop(1).content.toSeq shouldBe Seq(IntPair(42, 99)) } } - it should "filterNot" in { + it should "dropWhile" in { import IntPair._ matchIO(Table.parse(Seq("3 4", "1 2", "42 99"))) { case xt@HeadedTable(_, _) => - xt.filterNot(_.equals(IntPair(3, 4))).content.toSeq shouldBe Seq(IntPair(1, 2), IntPair(42, 99)) + xt.dropWhile(_.equals(IntPair(3, 4))).content.toSeq shouldBe Seq(IntPair(1, 2), IntPair(42, 99)) } } @@ -369,14 +364,14 @@ class TableSpec extends flatspec.AnyFlatSpec with should.Matchers { xt.slice(0, 2).content.toSeq shouldBe Seq(IntPair(3, 4), IntPair(1, 2)) } } -// -// it should "takeRight" in { -// import IntPair._ -// matchIO(Table.parse(Seq("3 4", "1 2", "42 99"))) { -// case xt@HeadedTable(_, _) => -// xt.takeRight(2).rows shouldBe Seq(IntPair(1, 2), IntPair(42, 99)) -// } -// } + + it should "take" in { + import IntPair._ + matchIO(Table.parse(Seq("3 4", "1 2", "42 99"))) { + case xt@HeadedTable(_, _) => + xt.take(2).content.toSeq shouldBe Seq(IntPair(3, 4), IntPair(1, 2)) + } + } it should "takeWhile" in { import IntPair._ @@ -408,8 +403,7 @@ class TableSpec extends flatspec.AnyFlatSpec with should.Matchers { implicit val r: HierarchicalRenderer[Indexed[IntPair]] = indexedRenderer("", "th") } - // FIXME this is a mystery - ignore should "render the table to CSV" in { + it should "render the table to CSV" in { import IntPair._ matchIO(Table.parse(Seq("1 2", "42 99"))) { case HeadedTable(_, _) => succeed @@ -448,7 +442,7 @@ class TableSpec extends flatspec.AnyFlatSpec with should.Matchers { case HeadedTable(_, _) => succeed } - val file = new File("output.csv") + val file = new File("tmp/other-render to CSV.csv") implicit val fw: Writable[FileWriter] = Writable.fileWritable(file) implicit object FileRenderer extends Renderer[Table[IntPair], FileWriter] { @@ -492,6 +486,7 @@ class TableSpec extends flatspec.AnyFlatSpec with should.Matchers { } implicit val csvAttributes: CsvAttributes = IntPairCsvRenderer.csvAttributes + implicit val randomIntPairOrdering: Ordering[IntPair] = Content.noOrdering[IntPair] matchIO(Table.parseFile(new File("src/test/resources/com/phasmidsoftware/table/intPairs.csv"))) { case iIt@HeadedTable(_, _) => val ws = iIt.toCSV @@ -515,6 +510,7 @@ class TableSpec extends flatspec.AnyFlatSpec with should.Matchers { def toColumnNames(wo: Option[String], no: Option[String]): String = s"a${csvAttributes.delimiter}b" } + implicit val randomIntPairOrdering: Ordering[IntPair] = Content.noOrdering[IntPair] matchIO(Table.parseFile(new File("src/test/resources/com/phasmidsoftware/table/intPairs.csv"))) { case iIt@HeadedTable(_, _) => val ws = iIt.toCSV @@ -693,6 +689,7 @@ class TableSpec extends flatspec.AnyFlatSpec with should.Matchers { val hdr = Header(Seq(Seq("a", "b"))) val row1 = Row(Seq("1", "2"), hdr, 1) val table = Table(Seq(row1), Some(hdr)) + implicit val randomRowOrdering: Ordering[Row] = Content.noOrdering[Row] EvaluateIO(Table.toCSVRow(table)) shouldBe "a,b\n1,2\n" } } diff --git a/src/test/scala/com/phasmidsoftware/util/EvaluateIO.scala b/src/test/scala/com/phasmidsoftware/util/EvaluateIO.scala index b6ee320d..fcc6abce 100644 --- a/src/test/scala/com/phasmidsoftware/util/EvaluateIO.scala +++ b/src/test/scala/com/phasmidsoftware/util/EvaluateIO.scala @@ -11,7 +11,9 @@ import scala.concurrent.Future import scala.util.{Failure, Success} /** - * Interim utilities for checking IO. + * Interim utilities for checking IO in a testing context. + * + * NOTE that these are based on Scalatest classes. * * Once we move to cats version 3, we will be able to use https://github.com/typelevel/cats-effect-testing */ diff --git a/src/test/scala/com/phasmidsoftware/write/WritableSpec.scala b/src/test/scala/com/phasmidsoftware/write/WritableSpec.scala index 0ab24c62..cc6be5c0 100644 --- a/src/test/scala/com/phasmidsoftware/write/WritableSpec.scala +++ b/src/test/scala/com/phasmidsoftware/write/WritableSpec.scala @@ -4,6 +4,7 @@ package com.phasmidsoftware.write +import com.phasmidsoftware.write.WritableSpec.complexFile import java.io.{File, FileWriter} import org.scalatest.flatspec import org.scalatest.matchers.should @@ -48,8 +49,8 @@ class WritableSpec extends flatspec.AnyFlatSpec with should.Matchers { |""".stripMargin } - it should "writeRow to a File" in { - val file = new File("output.csv") + it should "writeRowToFile" in { + val file = new File(complexFile) val fw: Writable[FileWriter] = Writable.fileWritable(file) val o = fw.unit fw.writeRow(o)(Complex(1, -1)) @@ -65,3 +66,7 @@ class WritableSpec extends flatspec.AnyFlatSpec with should.Matchers { } } + +object WritableSpec { + val complexFile = "tmp/Writable-writeRowToFile.csv" +} \ No newline at end of file diff --git a/tmp/Writable-writeRowToFile.csv b/tmp/Writable-writeRowToFile.csv new file mode 100644 index 00000000..bb97acd7 --- /dev/null +++ b/tmp/Writable-writeRowToFile.csv @@ -0,0 +1 @@ +1.0, -1.0