diff --git a/build.sbt b/build.sbt index 5ff81ed..8b90f13 100644 --- a/build.sbt +++ b/build.sbt @@ -8,6 +8,7 @@ val scala2 = List(scala2_12, scala2_13) val scala3 = List("3.3.8") val scalaTestVersion = "3.2.19" +val scalaTestPlusScalaCheckVersion = "3.2.19.0" excludeLintKeys in Global ++= Set(ideSkipProject) @@ -23,7 +24,8 @@ val commonSettings = commonSmlBuildSettings ++ ossPublishSettings ++ Seq( val commonJvmSettings = commonSettings ++ Seq( ideSkipProject := (scalaVersion.value != scala2_13), libraryDependencies ++= Seq( - "org.scalatest" %% "scalatest" % scalaTestVersion % Test + "org.scalatest" %% "scalatest" % scalaTestVersion % Test, + "org.scalatestplus" %% "scalacheck-1-18" % scalaTestPlusScalaCheckVersion % Test ), mimaPreviousArtifacts := previousStableVersion.value.map(organization.value %% moduleName.value % _).toSet, mimaReportBinaryIssues := { if ((publish / skip).value) {} else mimaReportBinaryIssues.value } @@ -47,14 +49,16 @@ val commonJsSettings = commonSettings ++ Seq( }, libraryDependencies ++= Seq( "org.scala-js" %%% "scalajs-dom" % "2.8.1", - "org.scalatest" %%% "scalatest" % scalaTestVersion % Test + "org.scalatest" %%% "scalatest" % scalaTestVersion % Test, + "org.scalatestplus" %%% "scalacheck-1-18" % scalaTestPlusScalaCheckVersion % Test ) ) val commonNativeSettings = commonSettings ++ Seq( ideSkipProject := true, libraryDependencies ++= Seq( - "org.scalatest" %%% "scalatest" % scalaTestVersion % Test + "org.scalatest" %%% "scalatest" % scalaTestVersion % Test, + "org.scalatestplus" %%% "scalacheck-1-18" % scalaTestPlusScalaCheckVersion % Test ) ) diff --git a/core/src/main/scala/sttp/model/sse/ServerSentEvent.scala b/core/src/main/scala/sttp/model/sse/ServerSentEvent.scala index 3919a58..b49671c 100644 --- a/core/src/main/scala/sttp/model/sse/ServerSentEvent.scala +++ b/core/src/main/scala/sttp/model/sse/ServerSentEvent.scala @@ -6,9 +6,37 @@ case class ServerSentEvent( data: Option[String] = None, eventType: Option[String] = None, id: Option[String] = None, - retry: Option[Int] = None + retry: Option[Int] = None, + comments: List[String] = Nil ) { + // required for binary compatibility + def this(data: Option[String], eventType: Option[String], id: Option[String], retry: Option[Int]) = + this(data, eventType, id, retry, Nil) + + def copy( + data: Option[String] = this.data, + eventType: Option[String] = this.eventType, + id: Option[String] = this.id, + retry: Option[Int] = this.retry, + comments: List[String] = this.comments + ): ServerSentEvent = ServerSentEvent(data, eventType, id, retry, comments) + + // required for binary compatibility + def copy( + data: Option[String], + eventType: Option[String], + id: Option[String], + retry: Option[Int] + ): ServerSentEvent = ServerSentEvent(data, eventType, id, retry, this.comments) + + /** True if the event carries no data, event type, id or retry. Clients ignore comments, so such events - keep-alive + * pings, but also blocks made up of unknown fields - carry nothing for the application and can usually be skipped. + */ + def hasNoFields: Boolean = data.isEmpty && eventType.isEmpty && id.isEmpty && retry.isEmpty + override def toString: String = { + val _comments = + comments.flatMap(ServerSentEvent.splitOnLineTerminators).map(comment => Some(s": $comment")).toArray val _data = data .map(ServerSentEvent.splitOnLineTerminators) .map(_.map(line => Some(s"data: $line"))) @@ -16,7 +44,9 @@ case class ServerSentEvent( val _event = eventType.map(event => s"event: ${ServerSentEvent.removeLineTerminators(event)}") val _id = id.map(id => s"id: ${ServerSentEvent.removeLineTerminators(id)}") val _retry = retry.map(retryCount => s"retry: $retryCount") - (_data :+ _event :+ _id :+ _retry).flatten.mkString("\n") + val _fields = _data :+ _event :+ _id :+ _retry + val _all = if (_comments.isEmpty) _fields else _comments ++ _fields + _all.flatten.mkString("\n") } } @@ -30,10 +60,26 @@ object ServerSentEvent { private def removeLineTerminators(s: String): String = if (s.indexOf('\r') < 0 && s.indexOf('\n') < 0) s else s.replaceAll(LineTerminators, "") + // required for binary compatibility + def apply( + data: Option[String], + eventType: Option[String], + id: Option[String], + retry: Option[Int] + ): ServerSentEvent = new ServerSentEvent(data, eventType, id, retry, Nil) + + /** An event consisting of comment lines only, one per line of the given text. Such events are ignored by clients, and + * can be used to keep the connection alive, so that it isn't dropped by proxies. + */ + def comment(text: String): ServerSentEvent = + ServerSentEvent(comments = splitOnLineTerminators(text).toList) + // https://html.spec.whatwg.org/multipage/server-sent-events.html def parse(event: List[String]): ServerSentEvent = { - event.foldLeft(ServerSentEvent()) { (event, line) => - if (line.startsWith("data:")) combineData(event, removeLeadingSpace(line.substring(5))) + // comments are prepended and reversed once at the end for performance + val parsed = event.foldLeft(ServerSentEvent()) { (event, line) => + if (line.startsWith(":")) event.copy(comments = removeLeadingSpace(line.substring(1)) :: event.comments) + else if (line.startsWith("data:")) combineData(event, removeLeadingSpace(line.substring(5))) else if (line.startsWith("id:")) event.copy(id = Some(removeLeadingSpace(line.substring(3)))) else if (line.startsWith("retry:")) event.copy(retry = ParseUtils.toIntOption(removeLeadingSpace(line.substring(6)))) @@ -43,14 +89,11 @@ object ServerSentEvent { else if (line == "event") event.copy(eventType = Some("")) else event } + if (parsed.comments.isEmpty) parsed else parsed.copy(comments = parsed.comments.reverse) } - private def combineData(event: ServerSentEvent, newData: String): ServerSentEvent = { - event match { - case e @ ServerSentEvent(Some(oldData), _, _, _) => e.copy(data = Some(s"$oldData\n$newData")) - case e @ ServerSentEvent(None, _, _, _) => e.copy(data = Some(newData)) - } - } + private def combineData(event: ServerSentEvent, newData: String): ServerSentEvent = + event.copy(data = Some(event.data.fold(newData)(oldData => s"$oldData\n$newData"))) private def removeLeadingSpace(s: String): String = if (s.startsWith(" ")) s.substring(1) else s } diff --git a/core/src/test/scala/sttp/model/sse/ServerSentEventPropertyTest.scala b/core/src/test/scala/sttp/model/sse/ServerSentEventPropertyTest.scala new file mode 100644 index 0000000..b0d7239 --- /dev/null +++ b/core/src/test/scala/sttp/model/sse/ServerSentEventPropertyTest.scala @@ -0,0 +1,99 @@ +package sttp.model.sse + +import org.scalacheck.Gen +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks + +class ServerSentEventPropertyTest extends AnyFlatSpec with ScalaCheckDrivenPropertyChecks with Matchers { + implicit override val generatorDrivenConfig: PropertyCheckConfiguration = + PropertyCheckConfiguration(minSuccessful = 100) + + private val LineTerminators = "\r\n|\r|\n" + + private val fieldValue: Gen[String] = Gen + .listOf( + Gen.frequency( + 6 -> Gen.oneOf("a", "b", ":", " "), + 2 -> Gen.oneOf("data: ", "event: ", "id: ", "retry: 9", "data", "id", "event", ": "), + 1 -> Gen.oneOf("\n", "\r", "\r\n") + ) + ) + .map(_.mkString) + + private val terminatorFreeValue: Gen[String] = + Gen.listOf(Gen.oneOf("a", "b", ":", " ", "data: ", "event: ", "id: ", "data")).map(_.mkString) + + private def eventsOf(value: Gen[String]): Gen[ServerSentEvent] = for { + data <- Gen.option(value) + eventType <- Gen.option(value) + id <- Gen.option(value) + retry <- Gen.option(Gen.chooseNum(Int.MinValue, Int.MaxValue)) + comments <- Gen.listOf(value) + } yield ServerSentEvent(data, eventType, id, retry, comments) + + private val events = eventsOf(fieldValue) + private val terminatorFreeEvents = eventsOf(terminatorFreeValue) + + private val allowedPrefixes = List("data:", "event:", "id:", "retry:", ":") + + private def lines(serialised: String): List[String] = serialised.split(LineTerminators, -1).toList + + private def lineCount(s: String): Int = s.split(LineTerminators, -1).length + + it should "serialise every line as a comment or a known field" in { + forAll(events) { sse => + val serialised = sse.toString + if (serialised.nonEmpty) { + lines(serialised).foreach { line => + withClue(s"line [$line] of [$serialised]: ") { + allowedPrefixes.exists(line.startsWith) shouldBe true + } + } + } + } + } + + it should "serialise one line per comment line, per data line and per other field that is set" in { + forAll(events) { sse => + val expected = sse.comments.map(lineCount).sum + sse.data.fold(0)(lineCount) + + List(sse.eventType, sse.id, sse.retry).count(_.isDefined) + lines(sse.toString).size shouldBe math.max(expected, 1) + } + } + + it should "serialise, parse and serialise again to the same result" in { + forAll(events) { sse => + val serialised = sse.toString + ServerSentEvent.parse(lines(serialised)).toString shouldBe serialised + } + } + + it should "parse back exactly what was serialised, when no value contains a line terminator" in { + forAll(terminatorFreeEvents) { sse => + ServerSentEvent.parse(lines(sse.toString)) shouldBe sse + } + } + + it should "parse any lines without throwing" in { + val anyLine = Gen.oneOf( + fieldValue, + Gen.oneOf( + "", + ":", + "data:", + "data", + "id:", + "id", + "event:", + "event", + "retry:", + "retry", + "retry: x", + "retry: 99999999999999999999", + "foo: bar" + ) + ) + forAll(Gen.listOf(anyLine)) { ls => noException should be thrownBy ServerSentEvent.parse(ls) } + } +} diff --git a/core/src/test/scala/sttp/model/sse/ServerSentEventTest.scala b/core/src/test/scala/sttp/model/sse/ServerSentEventTest.scala index dc36118..bc8cb1a 100644 --- a/core/src/test/scala/sttp/model/sse/ServerSentEventTest.scala +++ b/core/src/test/scala/sttp/model/sse/ServerSentEventTest.scala @@ -5,7 +5,7 @@ import org.scalatest.matchers.should.Matchers class ServerSentEventTest extends AnyFlatSpec with Matchers { val data = List( - (List(": this is a test stream"), ServerSentEvent()), + (List(": this is a test stream"), ServerSentEvent(comments = List("this is a test stream"))), (List("data: some text"), ServerSentEvent(Some("some text"))), (List("data: some text"), ServerSentEvent(Some(" some text"))), (List("data: another message", "data: with two lines"), ServerSentEvent(Some("another message\nwith two lines"))), @@ -24,7 +24,20 @@ class ServerSentEventTest extends AnyFlatSpec with Matchers { ( List("data: event1 data", "event: event1", "id: id1", "retry: 5"), ServerSentEvent(Some("event1 data"), Some("event1"), Some("id1"), Some(5)) - ) + ), + ( + List(": first", "data: x", ": second"), + ServerSentEvent(Some("x"), comments = List("first", "second")) + ), + ( + List(": one", "data: x", ": two", ": three"), + ServerSentEvent(Some("x"), comments = List("one", "two", "three")) + ), + (List(":no leading space"), ServerSentEvent(comments = List("no leading space"))), + (List(":"), ServerSentEvent(comments = List(""))), + (List("foo: bar", "data: x"), ServerSentEvent(Some("x"))), + (List("data"), ServerSentEvent(Some(""))), + (List("event"), ServerSentEvent(eventType = Some(""))) ) for ((lines, expected) <- data) { @@ -65,6 +78,148 @@ class ServerSentEventTest extends AnyFlatSpec with Matchers { |data: some data info 3""".stripMargin } + "composeSSE" should "serialise a comment-only event" in { + ServerSentEvent(comments = List("ping")).toString shouldBe ": ping" + } + + "composeSSE" should "serialise comments before the other fields" in { + val sse = ServerSentEvent(Some("d"), comments = List("c1", "c2")) + sse.toString shouldBe + s""": c1 + |: c2 + |data: d""".stripMargin + } + + "parse" should "round-trip an event with comments and all other fields set" in { + val sse = ServerSentEvent(Some("line1\nline2"), Some("evt"), Some("id1"), Some(5), List("c1", "c2")) + ServerSentEvent.parse(sse.toString.split("\n").toList) shouldBe sse + } + + "comment" should "create an event carrying a single comment" in { + ServerSentEvent.comment("ping") shouldBe ServerSentEvent(comments = List("ping")) + } + + "copy" should "preserve comments when another field is changed" in { + ServerSentEvent(comments = List("ping")).copy(data = Some("d")) shouldBe + ServerSentEvent(Some("d"), comments = List("ping")) + } + + "copy" should "preserve comments when given only the four original fields" in { + ServerSentEvent(comments = List("ping")).copy(Some("d"), None, None, None) shouldBe + ServerSentEvent(Some("d"), comments = List("ping")) + } + + "the constructor taking the four original fields" should "create an event without comments" in { + new ServerSentEvent(Some("d"), None, None, None) shouldBe ServerSentEvent(Some("d")) + } + + "composeSSE" should "serialise a multi-line comment as multiple comment lines" in { + ServerSentEvent.comment("a\nb").toString shouldBe + s""": a + |: b""".stripMargin + } + + "composeSSE" should "emit empty comment lines, not blank lines, for a comment ending with newlines" in { + ServerSentEvent.comment("ping\n\n").toString shouldBe ": ping\n: \n: " + } + + "composeSSE" should "emit empty comment lines for a comment of line terminators only" in { + ServerSentEvent.comment("\n").toString shouldBe ": \n: " + } + + "composeSSE" should "emit a single empty comment line for an empty comment" in { + ServerSentEvent.comment("").toString shouldBe ": " + } + + "composeSSE" should "serialise a comment containing a carriage return as multiple comment lines" in { + ServerSentEvent.comment("x\rdata: y").toString shouldBe + s""": x + |: data: y""".stripMargin + } + + "composeSSE" should "serialise a comment containing CRLF as multiple comment lines" in { + ServerSentEvent.comment("a\r\nb").toString shouldBe + s""": a + |: b""".stripMargin + } + + "comment" should "not allow a carriage return to inject other fields" in { + val sse = ServerSentEvent.comment("x\rdata: y") + ServerSentEvent.parse(sse.toString.split("\r\n|\r|\n").toList) shouldBe + ServerSentEvent(comments = List("x", "data: y")) + } + + "comment" should "not allow a newline to inject other fields" in { + val sse = ServerSentEvent.comment("x\ndata: y") + ServerSentEvent.parse(sse.toString.split("\n").toList) shouldBe + ServerSentEvent(comments = List("x", "data: y")) + } + + "apply" should "keep comments as they were given" in { + ServerSentEvent(comments = List("a\nb")).comments shouldBe List("a\nb") + } + + "comment" should "split a multi-line comment into separate comments" in { + ServerSentEvent.comment("a\nb\rc\r\nd").comments shouldBe List("a", "b", "c", "d") + } + + "copy" should "keep comments as they were given" in { + ServerSentEvent().copy(comments = List("a\nb")).comments shouldBe List("a\nb") + } + + "parse" should "round-trip an event built from a multi-line comment" in { + val sse = ServerSentEvent.comment("a\nb") + ServerSentEvent.parse(sse.toString.split("\r\n|\r|\n").toList) shouldBe sse + } + + val roundTripComments = List( + List("ping"), + List(""), + List("\n"), + List("ping\n\n"), + List("a\nb"), + List("x\rdata: y"), + List("a\n\nb"), + List(" spaced"), + List("", "b") + ) + + for (comments <- roundTripComments) { + "parse" should s"round-trip comments ${comments.map(_.replace("\r", "\\r").replace("\n", "\\n"))}" in { + val sse = ServerSentEvent(Some("d1\nd2"), Some("evt"), Some("id1"), Some(7), comments) + val serialised = sse.toString + ServerSentEvent.parse(serialised.split("\r\n|\r|\n").toList).toString shouldBe serialised + } + } + + "hasNoFields" should "be true for a keep-alive event" in { + ServerSentEvent.comment("ping").hasNoFields shouldBe true + } + + "hasNoFields" should "be true for an empty event" in { + ServerSentEvent().hasNoFields shouldBe true + } + + "hasNoFields" should "be true for an event of unknown fields only" in { + ServerSentEvent.parse(List("foo: bar")).hasNoFields shouldBe true + } + + "hasNoFields" should "be false when data is set" in { + ServerSentEvent(Some("d"), comments = List("ping")).hasNoFields shouldBe false + } + + "hasNoFields" should "be false when only the event type is set" in { + ServerSentEvent(eventType = Some("e")).hasNoFields shouldBe false + } + + "hasNoFields" should "be false when only the id is set" in { + ServerSentEvent(id = Some("i")).hasNoFields shouldBe false + } + + "hasNoFields" should "be false when only retry is set" in { + ServerSentEvent(retry = Some(5)).hasNoFields shouldBe false + } + "composeSSE" should "split data on all line terminators" in { val sse = ServerSentEvent(Some("line 1\r\nline 2\rline 3\nline 4"))