Add Optional Sub Second precision Date/Instant Support (#94)

* initial for testing

* remove old tests

* Add support for Stix optional subsecond timestamp support with nano-second support

* Add StixInstance class and related support changes to control subsecond precision

* Remove println from test

* fix incorrect import for Process.class

* version bump

* Update Date Precision support notes

Author: StephenOTT

README.md

@@ -25,6 +25,22 @@ https://github.com/StephenOTT/TAXII-springboot-bpmn
 
 If you are looking for a gson based implementation, [CS-AWARE](https://cs-aware.eu) provides a [gson based implementation of the Stix2 library](https://github.com/cs-aware/stix2).
 
+
+## Sub-Second Precision Support
+
+STIX-Java supports zero to 9 digit nanosecond precision with any date that is parsed by STIX-Java.  
+This means that while a date that was sourced/generated by the STIX-Java library will be at sub-second precision of 3 digits, 
+if you are parsing JSON with greater precision, or you supply a custom Instant with greater precision for a specific field, 
+STIX-Java will support this and store the extra precision.
+
+General rules to understand:
+
+1. By default, timestamps generated natively by the STIX-Java library will be be with 3 digits of sub-second precision (millisecond precision).
+1. Sub-second precision from 0 to 9 digits (9 digits its nano second precision: `hh:mm:ss.999999999`) is supported.  This means you can omit sub-seconds if you choose.
+1. The StixInstant.class supports a `toString()` method that will generate a STIX Spec Date with the original precision preserved
+1. Json parsing of JSON strings will support all of the above rules. 
+
+
 -----
 
 ## Java Usage
@@ -217,8 +233,7 @@ class SomeClass {
     Optional<IdentitySdo> getCreatedByRef();
 
     @NotNull
-    @JsonSerialize(using = InstantSerializer.class)
-    @JsonFormat(shape=JsonFormat.Shape.STRING, pattern = StixDataFormats.TIMESTAMP_PATTERN, timezone = "UTC")
+    @JsonSerialize(using = StixInstantSerializer.class) @JsonDeserialize(using = StixInstantDeserializer.class)
     @JsonProperty("created")
     @Value.Default
     @Redactable(useMask = true)

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>io.digitalstate.stix</groupId>
     <artifactId>stix</artifactId>
-    <version>v0.7.2</version>
+    <version>v0.7.3</version>
     <packaging>jar</packaging>
 
     <name>STIX 2</name>

src/main/java/io/digitalstate/stix/common/StixCommonProperties.java

@@ -3,10 +3,12 @@
 import com.fasterxml.jackson.annotation.*;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.digitalstate.stix.bundle.BundleableObject;
 import io.digitalstate.stix.datamarkings.GranularMarkingDm;
 import io.digitalstate.stix.datamarkings.MarkingDefinitionDm;
-import io.digitalstate.stix.helpers.StixDataFormats;
+import io.digitalstate.stix.json.StixInstantDeserializer;
+import io.digitalstate.stix.json.StixInstantSerializer;
 import io.digitalstate.stix.json.StixParsers;
 import io.digitalstate.stix.json.converters.dehydrated.DomainObjectOptionalConverter;
 import io.digitalstate.stix.json.converters.dehydrated.MarkingDefinitionSetConverter;
@@ -23,7 +25,6 @@
 import javax.validation.constraints.Pattern;
 import javax.validation.constraints.Size;
 import javax.validation.groups.Default;
-import java.time.Instant;
 import java.util.Optional;
 import java.util.Set;
 
@@ -70,13 +71,13 @@ default boolean getHydrated(){
     Optional<IdentitySdo> getCreatedByRef();
 
     @NotNull
-    @JsonFormat(shape=JsonFormat.Shape.STRING, pattern = StixDataFormats.TIMESTAMP_PATTERN, timezone = "UTC")
+    @JsonSerialize(using = StixInstantSerializer.class) @JsonDeserialize(using = StixInstantDeserializer.class)
     @JsonProperty("created")
     @JsonPropertyDescription("The created property represents the time at which the first version of this object was created. The timstamp value MUST be precise to the nearest millisecond.")
     @Value.Default
     @Redactable(useMask = true)
-    default Instant getCreated(){
-        return Instant.now();
+    default StixInstant getCreated(){
+        return new StixInstant();
     }
 
     @NotNull

src/main/java/io/digitalstate/stix/common/StixInstant.java

@@ -0,0 +1,93 @@
+package io.digitalstate.stix.common;
+
+import io.digitalstate.stix.helpers.StixDataFormats;
+
+import java.time.Instant;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Stores a Java.time Instant and stores the original sub-second precision digital count.
+ * SubSecond precision count means that the number of digits (even if all zeros / trailing zeros) are remembered.
+ * Use toString() to get the Stix string format for the datetime.
+ */
+public class StixInstant {
+
+    public static final Pattern REGEX_SUBSECOND = Pattern.compile("(?<fullDate>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(.(?<subSecond>[0-9]+))?Z)");
+
+    private final Instant instant;
+    private final int originalSubSecondPrecisionDigitCount;
+
+    /**
+     * Generates a Instant.now()
+     */
+    public StixInstant() {
+        Instant now = Instant.now();
+        this.instant = now;
+        this.originalSubSecondPrecisionDigitCount = 3;
+    }
+
+    public StixInstant(Instant instant, int subSecondPrecision) {
+        Objects.requireNonNull(instant);
+
+        this.instant = instant;
+        this.originalSubSecondPrecisionDigitCount = subSecondPrecision;
+    }
+
+    /**
+     * Defaults with parsed Instant's Nano second precision digit count.
+     * Note that trailing zeros would have been stripped.
+     * If you need to keep exact precision, including the trailing zeros, use {@link StixInstant#StixInstant(Instant, int)}
+     * @param instant
+     */
+    public StixInstant(Instant instant) {
+        this(instant, String.valueOf(instant.getNano()).length());
+    }
+
+    /**
+     * Get the underlying Instant value.  If you need the Stix Date with the Stix Precision then use {@link StixInstant#toString()}
+     * getInstant() should only be used in special cases where you need access to perform temporal work.
+     * @return
+     */
+    public Instant getInstant() {
+        return instant;
+    }
+
+    public int getOriginalSubSecondPrecisionDigitCount() {
+        return originalSubSecondPrecisionDigitCount;
+    }
+
+    /**
+     * Generates a STIX Spec String of DateTime.
+     * Uses {@link StixDataFormats#getWriterStixDateTimeFormatter(int)}
+     * @return
+     */
+    @Override
+    public String toString() {
+        return StixDataFormats.getWriterStixDateTimeFormatter(originalSubSecondPrecisionDigitCount)
+                .format(this.instant);
+    }
+
+    public static StixInstant parse(String dateString){
+        Instant instant = Instant.from(StixDataFormats.getReaderStixDateTimeFormatter().parse(dateString));
+        int subSecondCount = getSubSecondDigitCount(dateString);
+        return new StixInstant(instant, subSecondCount);
+    }
+
+    private static int getSubSecondDigitCount(String dateString){
+        Matcher matcher = REGEX_SUBSECOND.matcher(dateString);
+        if (matcher.find()){
+            String subSeconds = matcher.group("subSecond");
+            // If no sub seconds were provided then return 0 as the precision
+            if (subSeconds == null){
+                return 0;
+            } else {
+                return subSeconds.length();
+            }
+
+        } else {
+            throw new IllegalStateException("Unable to parse date");
+        }
+    }
+}

src/main/java/io/digitalstate/stix/common/StixModified.java

@@ -1,9 +1,11 @@
 package io.digitalstate.stix.common;
 
-import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyDescription;
-import io.digitalstate.stix.helpers.StixDataFormats;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import io.digitalstate.stix.json.StixInstantDeserializer;
+import io.digitalstate.stix.json.StixInstantSerializer;
 import io.digitalstate.stix.redaction.Redactable;
 import org.immutables.value.Value;
 
@@ -19,10 +21,10 @@ public interface StixModified {
     @NotNull
     @JsonProperty("modified")
     @JsonPropertyDescription("The modified property represents the time that this particular version of the object was created. The timstamp value MUST be precise to the nearest millisecond.")
-    @JsonFormat(pattern = StixDataFormats.TIMESTAMP_PATTERN, timezone = "UTC")
+    @JsonSerialize(using = StixInstantSerializer.class) @JsonDeserialize(using = StixInstantDeserializer.class)
     @Value.Default
     @Redactable
-    default Instant getModified(){
-        return Instant.now();
+    default StixInstant getModified(){
+        return new StixInstant();
     }
 }

src/main/java/io/digitalstate/stix/coo/extension/types/NtfsFileExtenstionExt.java

@@ -6,8 +6,8 @@
 import io.digitalstate.stix.coo.extension.CyberObservableExtension;
 import io.digitalstate.stix.coo.objects.FileCoo;
 import io.digitalstate.stix.coo.types.NtfsAlternateDataStreamObj;
-import io.digitalstate.stix.validation.contraints.coo.allowedparents.AllowedParents;
 import io.digitalstate.stix.validation.contraints.businessrule.BusinessRule;
+import io.digitalstate.stix.validation.contraints.coo.allowedparents.AllowedParents;
 import io.digitalstate.stix.validation.contraints.defaulttypevalue.DefaultTypeValue;
 import io.digitalstate.stix.validation.groups.DefaultValuesProcessor;
 import org.immutables.serial.Serial;

src/main/java/io/digitalstate/stix/coo/extension/types/PdfFileExtensionExt.java

@@ -5,8 +5,8 @@
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.digitalstate.stix.coo.extension.CyberObservableExtension;
 import io.digitalstate.stix.coo.objects.FileCoo;
-import io.digitalstate.stix.validation.contraints.coo.allowedparents.AllowedParents;
 import io.digitalstate.stix.validation.contraints.businessrule.BusinessRule;
+import io.digitalstate.stix.validation.contraints.coo.allowedparents.AllowedParents;
 import io.digitalstate.stix.validation.contraints.defaulttypevalue.DefaultTypeValue;
 import io.digitalstate.stix.validation.groups.DefaultValuesProcessor;
 import org.immutables.serial.Serial;

src/main/java/io/digitalstate/stix/coo/extension/types/RasterImageFileExtensionExt.java

@@ -12,7 +12,6 @@
 import org.immutables.serial.Serial;
 import org.immutables.value.Value;
 
-import javax.validation.Valid;
 import java.util.Map;
 import java.util.Optional;
 

src/main/java/io/digitalstate/stix/coo/extension/types/TcpExtensionExt.java

@@ -5,8 +5,8 @@
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.digitalstate.stix.coo.extension.CyberObservableExtension;
 import io.digitalstate.stix.coo.objects.NetworkTrafficCoo;
-import io.digitalstate.stix.validation.contraints.coo.allowedparents.AllowedParents;
 import io.digitalstate.stix.validation.contraints.businessrule.BusinessRule;
+import io.digitalstate.stix.validation.contraints.coo.allowedparents.AllowedParents;
 import io.digitalstate.stix.validation.contraints.defaulttypevalue.DefaultTypeValue;
 import io.digitalstate.stix.validation.groups.DefaultValuesProcessor;
 import org.immutables.serial.Serial;

src/main/java/io/digitalstate/stix/coo/extension/types/UnixAccountExtensionExt.java

@@ -5,8 +5,8 @@
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.digitalstate.stix.coo.extension.CyberObservableExtension;
 import io.digitalstate.stix.coo.objects.UserAccountCoo;
-import io.digitalstate.stix.validation.contraints.coo.allowedparents.AllowedParents;
 import io.digitalstate.stix.validation.contraints.businessrule.BusinessRule;
+import io.digitalstate.stix.validation.contraints.coo.allowedparents.AllowedParents;
 import io.digitalstate.stix.validation.contraints.defaulttypevalue.DefaultTypeValue;
 import io.digitalstate.stix.validation.groups.DefaultValuesProcessor;
 import org.immutables.serial.Serial;

src/main/java/io/digitalstate/stix/coo/extension/types/WindowsPeBinaryFileExtensionExt.java

@@ -3,11 +3,13 @@
 import com.fasterxml.jackson.annotation.*;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import io.digitalstate.stix.common.StixInstant;
 import io.digitalstate.stix.coo.extension.CyberObservableExtension;
 import io.digitalstate.stix.coo.objects.FileCoo;
 import io.digitalstate.stix.coo.types.WindowsPeOptionalHeaderObj;
 import io.digitalstate.stix.coo.types.WindowsPeSectionObj;
-import io.digitalstate.stix.helpers.StixDataFormats;
+import io.digitalstate.stix.json.StixOptionalInstantDeserializer;
+import io.digitalstate.stix.json.StixOptionalInstantSerializer;
 import io.digitalstate.stix.validation.contraints.coo.allowedparents.AllowedParents;
 import io.digitalstate.stix.validation.contraints.defaulttypevalue.DefaultTypeValue;
 import io.digitalstate.stix.validation.contraints.hashingvocab.HashingVocab;
@@ -66,8 +68,8 @@ public interface WindowsPeBinaryFileExtensionExt extends CyberObservableExtensio
 
     @JsonProperty("time_date_stamp")
     @JsonPropertyDescription("Specifies the time when the PE binary was created. The timestamp value MUST BE precise to the second.")
-    @JsonFormat(shape=JsonFormat.Shape.STRING, pattern = StixDataFormats.TIMESTAMP_PATTERN, timezone = "UTC")
-    Optional<Instant> getTimeDateStamp();
+    @JsonSerialize(using = StixOptionalInstantSerializer.class) @JsonDeserialize(using = StixOptionalInstantDeserializer.class)
+    Optional<StixInstant> getTimeDateStamp();
 
     @JsonProperty("pointer_to_symbol_table_hex")
     @JsonPropertyDescription("Specifies the file offset of the COFF symbol table.")

src/main/java/io/digitalstate/stix/coo/extension/types/WindowsProcessExtensionExt.java

@@ -5,8 +5,8 @@
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.digitalstate.stix.coo.extension.CyberObservableExtension;
 import io.digitalstate.stix.coo.objects.ProcessCoo;
-import io.digitalstate.stix.validation.contraints.coo.allowedparents.AllowedParents;
 import io.digitalstate.stix.validation.contraints.businessrule.BusinessRule;
+import io.digitalstate.stix.validation.contraints.coo.allowedparents.AllowedParents;
 import io.digitalstate.stix.validation.contraints.defaulttypevalue.DefaultTypeValue;
 import io.digitalstate.stix.validation.groups.DefaultValuesProcessor;
 import org.immutables.serial.Serial;

src/main/java/io/digitalstate/stix/coo/extension/types/WindowsServiceExtensionExt.java

@@ -16,7 +16,6 @@
 import org.immutables.value.Value;
 
 import javax.validation.constraints.NotBlank;
-import javax.validation.constraints.NotNull;
 import java.util.Optional;
 import java.util.Set;
 

src/main/java/io/digitalstate/stix/coo/objects/DirectoryCoo.java

@@ -3,8 +3,10 @@
 import com.fasterxml.jackson.annotation.*;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import io.digitalstate.stix.common.StixInstant;
 import io.digitalstate.stix.coo.CyberObservableObject;
-import io.digitalstate.stix.helpers.StixDataFormats;
+import io.digitalstate.stix.json.StixOptionalInstantDeserializer;
+import io.digitalstate.stix.json.StixOptionalInstantSerializer;
 import io.digitalstate.stix.validation.contraints.defaulttypevalue.DefaultTypeValue;
 import io.digitalstate.stix.validation.groups.DefaultValuesProcessor;
 import org.immutables.serial.Serial;
@@ -44,20 +46,20 @@ public interface DirectoryCoo extends CyberObservableObject {
     Optional<@Pattern(regexp = "^[a-zA-Z0-9/\\.+_:-]{2,250}$")
             String> getPathEnc();
 
-    @JsonFormat(pattern = StixDataFormats.TIMESTAMP_PATTERN, timezone = "UTC")
+    @JsonSerialize(using = StixOptionalInstantSerializer.class) @JsonDeserialize(using = StixOptionalInstantDeserializer.class)
     @JsonProperty("created")
     @JsonPropertyDescription("Specifies the date/time the directory was created.")
-    Optional<Instant> getCreated();
+    Optional<StixInstant> getCreated();
 
-    @JsonFormat(pattern = StixDataFormats.TIMESTAMP_PATTERN, timezone = "UTC")
+    @JsonSerialize(using = StixOptionalInstantSerializer.class) @JsonDeserialize(using = StixOptionalInstantDeserializer.class)
     @JsonProperty("modified")
     @JsonPropertyDescription("Specifies the date/time the directory was last written to/modified.")
-    Optional<Instant> getModified();
+    Optional<StixInstant> getModified();
 
-    @JsonFormat(pattern = StixDataFormats.TIMESTAMP_PATTERN, timezone = "UTC")
+    @JsonSerialize(using = StixOptionalInstantSerializer.class) @JsonDeserialize(using = StixOptionalInstantDeserializer.class)
     @JsonProperty("accessed")
     @JsonPropertyDescription("Specifies the date/time the directory was last accessed.")
-    Optional<Instant> getAccessed();
+    Optional<StixInstant> getAccessed();
 
     //@TODO add proper support for contains refs.  Must be Set of File or Directory types
     @JsonProperty("contains_refs")

src/main/java/io/digitalstate/stix/coo/objects/EmailMessageCoo.java

@@ -3,16 +3,17 @@
 import com.fasterxml.jackson.annotation.*;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import io.digitalstate.stix.common.StixInstant;
 import io.digitalstate.stix.coo.CyberObservableObject;
 import io.digitalstate.stix.coo.types.MimePartTypeObj;
-import io.digitalstate.stix.helpers.StixDataFormats;
+import io.digitalstate.stix.json.StixOptionalInstantDeserializer;
+import io.digitalstate.stix.json.StixOptionalInstantSerializer;
 import io.digitalstate.stix.validation.contraints.businessrule.BusinessRule;
 import io.digitalstate.stix.validation.contraints.defaulttypevalue.DefaultTypeValue;
 import io.digitalstate.stix.validation.groups.DefaultValuesProcessor;
 import org.immutables.serial.Serial;
 import org.immutables.value.Value;
 
-import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
 import java.time.Instant;
 import java.util.Map;
@@ -42,10 +43,10 @@ public interface EmailMessageCoo extends CyberObservableObject {
     @NotNull
     boolean isMultipart();
 
-    @JsonFormat(pattern = StixDataFormats.TIMESTAMP_PATTERN, timezone = "UTC")
+    @JsonSerialize(using = StixOptionalInstantSerializer.class) @JsonDeserialize(using = StixOptionalInstantDeserializer.class)
     @JsonProperty("date")
     @JsonPropertyDescription("Specifies the date/time that the email message was sent.")
-    Optional<Instant> getDate();
+    Optional<StixInstant> getDate();
 
     @JsonProperty("content_type")
     @JsonPropertyDescription("Specifies the value of the 'Content-Type' header of the email message.")