Skip to content

Write the usage sharing record as the specification says #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jwrosewell merged 7 commits into from
Nov 9, 2026
Merged

Write the usage sharing record as the specification says #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9885e36
Test the usage sharing record against the specification
jwrosewell Nov 9, 2026
7ad2741
Write the usage sharing record as the specification says
jwrosewell Nov 9, 2026
c4e27a9
Merge rc into the usage sharing record draft
jwrosewell Nov 9, 2026
17d0c30
Test the usage sharing record's escaped characters and 10,000 charact…
jwrosewell Nov 9, 2026
6d4ab93
Write invalid characters as an escape and cut values at 10,000 charac…
jwrosewell Nov 9, 2026
38c3353
Test that the 10,000 character limit counts the value before its char…
jwrosewell Nov 9, 2026
5a356b7
Merge rc into the usage sharing record draft
jwrosewell Nov 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ...nes.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@
public class Constants {

/**
* The maximum length of a piece of evidence's value which can be added to
* the usage data being sent.
* The maximum length, in characters, of a piece of evidence's value which
* can be added to the usage data being sent.
*/
public static final int SHARE_USAGE_MAX_EVIDENCE_LENGTH = 512;
public static final int SHARE_USAGE_MAX_EVIDENCE_LENGTH = 10000;

/**
* Session id evidence suffix. This is prefixed by the query constant.
Expand Down
93 changes: 49 additions & 44 deletions ...yone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,6 @@
import java.net.URI;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import java.util.zip.GZIPOutputStream;

import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT;
Expand Down Expand Up @@ -422,7 +419,13 @@ private void streamXml(List<ShareUsageData> allData, OutputStream os) throws XML
*/
protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws XMLStreamException {
writer.writeStartElement("Device");
// --- write invariant data
// The elements follow the order of the specification's example.
// The SessionID used to track a series of requests
writeXmlElement(writer, "SessionId", data.sessionId);
// The sequence number of the request in a series of requests.
writeXmlElement(writer, "Sequence", String.valueOf(data.sequence));
// The UTC date/time this entry was written
writeXmlElement(writer, "DateSent", DATE_FMT.format(new Date()));
// The version number of the Pipeline API
writeXmlElement(writer, "Version", coreVersion);
// Write Pipeline information
Expand All @@ -435,21 +438,13 @@ protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws
writeXmlElement(writer, "Language", "java");
// The software language version
writeXmlElement(writer, "LanguageVersion", languageVersion);
// The client IP of the request
writeXmlElement(writer, "ClientIP", data.clientIP);
// The IP of this server
writeXmlElement(writer, "ServerIP", getHostAddress());
// The OS name and version
writeXmlElement(writer, "Platform", osVersion);

// --- write variable data
// The SessionID used to track a series of requests
writeXmlElement(writer, "SessionId", data.sessionId);
// The sequence number of the request in a series of requests.
writeXmlElement(writer, "Sequence", String.valueOf(data.sequence));
// The UTC date/time this entry was written
writeXmlElement(writer, "DateSent", DATE_FMT.format(new Date()));
// The client IP of the request
writeXmlElement(writer, "ClientIP", data.clientIP);

// Write all other evidence data that has been included.
for (Map.Entry<String, Map<String, String>> category : data.evidenceData.entrySet()) {
for (Map.Entry<String, String> entry : category.getValue().entrySet()) {
Expand All @@ -461,7 +456,7 @@ protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws
writer.writeStartElement(entry.getKey());
}
if (replacedString.isReplaced()) {
writer.writeAttribute("replaced", "true");
writer.writeAttribute("escaped", "true");
}
if (replacedString.isTruncated()) {
writer.writeAttribute("truncated", "true");
Expand All @@ -488,45 +483,55 @@ protected void writeXmlElement(XMLStreamWriter writer,
* replace characters that cause problems in XML with the "Replacement character"
*/
public static class ReplacedString {
// a set of valid XML character values (ignoring valid controls x09, x0a, x0d, x85)
static final Set<Integer> VALID_XML_CHARS =
Stream.of( IntStream.range(0x20,0x7F),
IntStream.range(0xA0, 0x100))
.flatMapToInt(i -> i)
.boxed()
.collect(Collectors.toCollection(HashSet::new));
// an array describing whether a character value is valid
static final Boolean[] IS_VALID_XML_CHAR = IntStream
.range(0, 0x100)
.mapToObj(VALID_XML_CHARS::contains)
.toArray(Boolean[]::new);

// maximum string length
// maximum string length, in characters
static final int MAX_LENGTH = SHARE_USAGE_MAX_EVIDENCE_LENGTH;

private boolean replaced;
private boolean truncated;
private final StringBuilder builder;

/**
* Whether the character is allowed in an XML 1.0 document, as the
* Char production in section 2.2 of the XML 1.0 specification says.
* @param codePoint the character to check
* @return true if the character is allowed in XML 1.0
*/
static boolean isXmlChar(int codePoint) {
return codePoint == 0x9 || codePoint == 0xA || codePoint == 0xD ||
(codePoint >= 0x20 && codePoint <= 0xD7FF) ||
(codePoint >= 0xE000 && codePoint <= 0xFFFD) ||
(codePoint >= 0x10000 && codePoint <= 0x10FFFF);
}

/**
* Keep the first {@link #MAX_LENGTH} characters of the text, and write
* each character XML 1.0 does not allow as a backslash, an x and the
* four lower-case hexadecimal digits of its code. The limit counts
* characters, so a pair of surrogates is never split.
* @param text the text to prepare
*/
public ReplacedString(String text) {
if (Objects.isNull(text) || text.length() == 0) {
builder = new StringBuilder(0);
// Using 'return' statement in a constructor throws compilation
// error in Jdk 8.
// return;
} else {
builder = new StringBuilder(Math.min(text.length(), MAX_LENGTH));
truncated = text.length() > MAX_LENGTH;
text.chars()
.limit(MAX_LENGTH)
.forEach(c -> {
if (c < IS_VALID_XML_CHAR.length && IS_VALID_XML_CHAR[c]) {
builder.append((char) c);
} else {
builder.append((char) 0xFFFD);
replaced = true;
}
});
int end = text.length();
if (text.codePointCount(0, end) > MAX_LENGTH) {
end = text.offsetByCodePoints(0, MAX_LENGTH);
truncated = true;
}
builder = new StringBuilder(end);
int index = 0;
while (index < end) {
int codePoint = text.codePointAt(index);
if (isXmlChar(codePoint)) {
builder.appendCodePoint(codePoint);
} else {
builder.append("\\x").append(
String.format("%04x", codePoint));
replaced = true;
}
index += Character.charCount(codePoint);
}
}
}

Expand Down
137 changes: 114 additions & 23 deletions ...one/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ReplacedStringTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,16 @@
*/
class ReplacedStringTest {

// is transparent to legal characters
// is transparent to the three control characters XML 1.0 allows
@Test
public void testNormal(){
String aString = IntStream.of(0x9, 0xA, 0xD)
.mapToObj(i -> Character.toString((char)i))
.collect(joining());
ReplacedString replacedString = new ReplacedString(aString);
assertFalse(replacedString.isTruncated());
assertTrue(replacedString.isReplaced());
assertEquals(3, replacedString.toString().length());
assertFalse(replacedString.isReplaced());
assertEquals(aString, replacedString.toString());
}
// is transparent to legal characters except \
@Test
Expand Down Expand Up @@ -83,14 +83,74 @@ public void testNull(){
assertEquals(0, escapedString.toString().length());
assertEquals("", escapedString.toString());
}
// truncates longer than 512
// truncates longer than 10,000 characters
@Test
public void testTruncate(){
String aString = IntStream.range(0,513).mapToObj(i -> "a").collect(joining());
String aString = repeat("a", 10001);
ReplacedString replacedString = new ReplacedString(aString);
assertTrue(replacedString.isTruncated());
assertFalse(replacedString.isReplaced());
assertEquals(10000, replacedString.toString().length());
}
// keeps a value of exactly 10,000 characters whole
@Test
public void testNotTruncatedAtTheLimit(){
String aString = repeat("a", 10000);
ReplacedString replacedString = new ReplacedString(aString);
assertFalse(replacedString.isTruncated());
assertEquals(aString, replacedString.toString());
}
// the limit counts characters, so a character made of a pair of
// surrogates at the limit is kept whole and the next one is cut
@Test
public void testTruncateNeverSplitsASurrogatePair(){
String emoji = "\uD83D\uDE00";
String aString = repeat("a", 9999) + emoji + "b";
ReplacedString replacedString = new ReplacedString(aString);
assertTrue(replacedString.isTruncated());
assertFalse(replacedString.isReplaced());
assertEquals(512, replacedString.toString().length());
assertEquals(repeat("a", 9999) + emoji, replacedString.toString());
}
// a pair that would start after the limit is cut with the rest
@Test
public void testTruncateBeforeASurrogatePair(){
String emoji = "\uD83D\uDE00";
String aString = repeat("a", 10000) + emoji;
ReplacedString replacedString = new ReplacedString(aString);
assertTrue(replacedString.isTruncated());
assertEquals(repeat("a", 10000), replacedString.toString());
}
// counts characters and not UTF-16 code units
@Test
public void testLimitCountsCharactersNotCodeUnits(){
String emoji = "\uD83D\uDE00";
ReplacedString atLimit = new ReplacedString(repeat(emoji, 10000));
assertFalse(atLimit.isTruncated());
assertEquals(20000, atLimit.toString().length());
ReplacedString over = new ReplacedString(repeat(emoji, 10001));
assertTrue(over.isTruncated());
assertEquals(repeat(emoji, 10000), over.toString());
}
// the limit counts the value as supplied, so the escapes written for
// invalid characters do not count towards it
@Test
public void testEscapesDoNotCountTowardsTheLimit(){
String aString = "a" + (char) 0x1 + repeat("b", 9998);
ReplacedString replacedString = new ReplacedString(aString);
assertFalse(replacedString.isTruncated());
assertTrue(replacedString.isReplaced());
assertEquals(10000 - 1 + 6, replacedString.toString().length());
// One character more is cut, whatever the escapes add.
ReplacedString over = new ReplacedString(aString + "c");
assertTrue(over.isTruncated());
assertEquals(10000 - 1 + 6, over.toString().length());
}
private static String repeat(String text, int times) {
StringBuilder builder = new StringBuilder();
for (int i = 0; i < times; i++) {
builder.append(text);
}
return builder.toString();
}
// escapes single digit Hex
@Test
Expand All @@ -99,8 +159,7 @@ public void testEscapedSmall(){
ReplacedString replacedString = new ReplacedString(aString);
assertFalse(replacedString.isTruncated());
assertTrue(replacedString.isReplaced());
System.out.println(replacedString);
assertEquals(5, replacedString.toString().length());
assertEquals(repeat("\\x0001", 5), replacedString.toString());
}
// escapes 2 digit hex
@Test
Expand All @@ -109,8 +168,7 @@ public void testEscapedBigger(){
ReplacedString replacedString = new ReplacedString(aString);
assertFalse(replacedString.isTruncated());
assertTrue(replacedString.isReplaced());
System.out.println(replacedString);
assertEquals(5, replacedString.toString().length());
assertEquals(repeat("\\x001f", 5), replacedString.toString());
}
// does not escape valid chars
@Test
Expand All @@ -123,15 +181,50 @@ public void testEscapedBiggerStill(){
assertEquals(5, replacedString.toString().length());
}

@Test ()
public void testEscapedEvenBigger(){
String aString = IntStream.range(0,5).mapToObj(i -> "\u0100").collect(joining());
// does not replace characters above U+00FF, which XML 1.0 allows
@Test
public void testNonLatinKept(){
String aString = "\u0100 \u0416 \u6F22\u5B57 \uD7FF \uE000 \uFFFD";
ReplacedString replacedString = new ReplacedString(aString);
assertFalse(replacedString.isTruncated());
assertTrue(replacedString.isReplaced());
System.out.println(replacedString);
assertEquals(5, replacedString.toString().length());

assertFalse(replacedString.isReplaced());
assertEquals(aString, replacedString.toString());
}
// does not replace the controls from U+007F to U+009F, which XML 1.0 allows
@Test
public void testC1ControlsKept(){
String aString = IntStream.range(0x7F, 0xA0)
.mapToObj(i -> Character.toString((char)i))
.collect(joining());
ReplacedString replacedString = new ReplacedString(aString);
assertFalse(replacedString.isReplaced());
assertEquals(aString, replacedString.toString());
}
// keeps a pair of surrogates, which is one character above U+FFFF
@Test
public void testSurrogatePairKept(){
String aString = "a\uD83D\uDE00b";
ReplacedString replacedString = new ReplacedString(aString);
assertFalse(replacedString.isReplaced());
assertEquals(aString, replacedString.toString());
}
// writes a surrogate on its own, U+FFFE and U+FFFF as \x and four
// lower-case hexadecimal digits
@Test
public void testForbiddenCharactersEscaped(){
String[][] cases = {
{"\uD83D", "\\xd83d"}, {"\uDE00", "\\xde00"},
{"\uFFFE", "\\xfffe"}, {"\uFFFF", "\\xffff"},
{"\u0000", "\\x0000"}, {"\u000B", "\\x000b"}};
for (String[] c : cases) {
ReplacedString replacedString = new ReplacedString("a" + c[0] + "b");
assertTrue(replacedString.isReplaced());
assertEquals("a" + c[1] + "b", replacedString.toString());
}
// A low surrogate before a high one is two surrogates on their own.
ReplacedString reversed = new ReplacedString("\uDE00\uD83D");
assertTrue(reversed.isReplaced());
assertEquals("\\xde00\\xd83d", reversed.toString());
}

private final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();
Expand All @@ -140,12 +233,12 @@ public void testEscapedEvenBigger(){
@Test
public void testXmlSerialise() throws XMLStreamException {
xmlOutputFactory.setProperty("escapeCharacters", true);
String text = ReplacedString.VALID_XML_CHARS.stream()
.map(i->Character.toString((char)i.intValue()))
.collect(joining());
String text = "\t\n a\u00E9\u0100\u0416\u6F22\u5B57\uD83D\uDE00" +
"\u007F\u0080\u009F\uD7FF\uE000\uFFFD <&>";
assertEquals(text, new ReplacedString(text).toString());
ByteArrayOutputStream os = new ByteArrayOutputStream();
XMLStreamWriter writer = xmlOutputFactory.createXMLStreamWriter(os, "UTF-8");
writer.writeStartDocument("UTF-8", "1.1");
writer.writeStartDocument("UTF-8", "1.0");
writer.writeStartElement("test");
writer.writeCharacters(text);
writer.writeEndElement();
Expand All @@ -155,8 +248,6 @@ public void testXmlSerialise() throws XMLStreamException {
XMLStreamReader reader = xmlInputFactory.createXMLStreamReader(is);
reader.next();
String readText = reader.getElementText();
System.out.println(text);
System.out.println(readText);
reader.close();

assertEquals(readText, text);
Expand Down
Loading
Loading