ti_threatconnect: add in technique.name field to the transform and re…

…move milliseconds from TQL query. (#11217)

* Add in technique.name field to the transform that came from a customer
  request.
* Remove milliseconds from TQL query as current code drops the last 0 in the
  milliseconds which causes as issue.

packages/ti_threatconnect/changelog.yml

@@ -1,9 +1,9 @@
 # newer versions go on top
-- version: "1.2.3"
+- version: "1.3.0"
   changes:
-    - description: Fix labels.is_ioc_transform_source values
-      type: bugfix
-      link: https://github.com/elastic/integrations/pull/11231
+    - description: Add in technique.name field to the transform. Remove milliseconds from TQL query.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11217
 - version: "1.2.2"
   changes:
     - description: Add missing fields in transform

packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json

@@ -92,6 +92,9 @@
                 "technique": {
                     "id": [
                         "T1055.005"
+                    ],
+                    "name": [
+                        "userexecution:maliciouslink"
                     ]
                 }
             },
@@ -537,4 +540,4 @@
             }
         }
     ]
-}
+}

packages/ti_threatconnect/data_stream/indicator/agent/stream/cel.yml.hbs

@@ -64,10 +64,12 @@ redact:
     - secret_key
 
 # The program section is where the logic of the stream processor is defined.
+# Notice the format for the last timestamp does not include milliseconds. The default format included
+# milliseconds and if it ended in 0, that 0 would be dropped and TC TQL then would error on the timestamp.
 program: |
   ['lastModified GEQ "'+(
       !state.want_more ?
-          state.?cursor.last_timestamp.orValue(string(now - duration(state.initial_interval)))
+          state.?cursor.last_timestamp.orValue((now - duration(state.initial_interval)).format(time_layout.RFC3339))
       :
           state.?cursor.first_timestamp.orValue("")
       )+'"'+(

packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml

@@ -1985,7 +1985,31 @@ processors:
             - remove:
                 field: _ingest._value.lastUsed
                 ignore_missing: true
-# Some tags are Attack Patterns, rename the techniqueId to technique.id
+  # Some tags are Attack Patterns, map technique name to ECS field if technique ID exists
+  - script:
+      lang: painless
+      description: "Extract Technique Name from tags"
+      if: ctx.json?.tags?.data != null
+      tag: script_technique_names
+      source: |-
+        def t_names = new ArrayList();
+        for (def obj : ctx.json.tags.data) {
+          if (obj.get('techniqueId') != null) {
+            t_names.add(obj.get('name'));
+          }
+        }
+        if (t_names.size() > 0){
+          if (ctx.threat.technique == null) {
+            ctx.threat.technique = new HashMap();
+          }
+          ctx.threat.technique.name = t_names;
+        }
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+
+# Some tags are Attack Patterns, rename the technique id field for ES standardization
   - foreach:
       field: json.tags.data
       if: ctx.json?.tags?.data != null
@@ -1996,7 +2020,7 @@ processors:
           tag: rename_tags_data_techniqueId
           target_field: _ingest._value.technique.id
           ignore_missing: true
-# Append the current attack pattern technique id to the threat.technique.id field to capture the full list
+# Some tags are Attack Patterns, map technique id to ECS field
   - foreach:
       field: json.tags.data
       if: ctx.json?.tags?.data != null
@@ -2007,6 +2031,7 @@ processors:
           tag: append_tags_data_techniqueId
           value: '{{{_ingest._value.technique.id}}}'
           allow_duplicates: false
+
 # For Attack pattern tags, capture the platform data and append to os.family
   - foreach:
       field: json.tags.data

packages/ti_threatconnect/elasticsearch/transform/latest/fields/ecs.yml

@@ -60,9 +60,11 @@
   type: keyword
 - name: threat.technique.id
   type: keyword
+- name: threat.technique.name
+  type: keyword
 - name: user_agent.original
   type: keyword
-# Below fields to be moved into base-fields.yml after kibana.version changed to >= 8.14 
+# Below fields to be moved into base-fields.yml after kibana.version changed to >= 8.14
 # Related to fix: https://github.com/elastic/kibana/pull/177608
 - name: event.module
   type: constant_keyword

packages/ti_threatconnect/manifest.yml

@@ -2,7 +2,7 @@
 format_version: 3.0.3
 name: ti_threatconnect
 title: ThreatConnect
-version: "1.2.3"
+version: "1.3.0"
 description: Collects Indicators from ThreatConnect using the Elastic Agent and saves them as logs inside Elastic
 type: integration
 categories: