Merge pull request #377 from GreyNoise-Intelligence/new_features_and_…

…requests New features and requests
GreyNoise-Intelligence · Dec 18, 2020 · bedf40f · bedf40f
2 parents 99c4647 + 5fdd4cc
commit bedf40f
Show file tree

Hide file tree

Showing 10 changed files with 149 additions and 12 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -6,17 +6,9 @@ Version `dev`_
 ================
 **Date**: unreleased
 
-* API client:
-
-  * 1
-
-* CLI:
-
-  * 1
-
 * Both API client and CLI:
 
-  * 1
+  * Fix IP_Validation method bug which was preventing valid IPs from being submitted
 
 Version `0.5.0`_
 ================

diff --git a/src/greynoise/api/__init__.py b/src/greynoise/api/__init__.py
@@ -40,6 +40,7 @@ class GreyNoise(object):
     EP_NOISE_MULTI = "noise/multi/quick"
     EP_NOISE_CONTEXT = "noise/context/{ip_address}"
     EP_META_METADATA = "meta/metadata"
+    EP_META_PING = "meta/ping"
     EP_NOT_IMPLEMENTED = "request/{subcommand}"
     UNKNOWN_CODE_MESSAGE = "Code message unknown: {}"
     CODE_MESSAGES = {
@@ -358,3 +359,12 @@ def metadata(self):
         LOGGER.debug("Getting metadata...")
         response = self._request(self.EP_META_METADATA)
         return response
+
+    def test_connection(self):
+        """Test the API connection and API key."""
+        LOGGER.debug("Testing access to GreyNoise API and for valid API Key")
+        try:
+            self._request(self.EP_META_PING)
+            return "Success: Access and API Key Valid"
+        except Exception as e:
+            return e
diff --git a/src/greynoise/cli/templates/gnql_query.txt.j2 b/src/greynoise/cli/templates/gnql_query.txt.j2
@@ -5,7 +5,7 @@ Query: {{ result.query }}
 {%- if result.data %}
 {%- for ip_context in result.data %}
 {{ macros.result_header(loop) }}
-{%- include "ip_context_result.txt.j2" %}
+{%- include "ip_context_result_query.txt.j2" %}
 {%- endfor %}
 {% else %}
 No results found for this query.

diff --git a/src/greynoise/cli/templates/ip_context_result.txt.j2 b/src/greynoise/cli/templates/ip_context_result.txt.j2
@@ -18,13 +18,22 @@
 <key>ASN</key>: <value>{{ ip_context.metadata.asn }}</value>
 <key>Category</key>: <value>{{ ip_context.metadata.category }}</value>
 <key>Location</key>: <value>{{ ip_context.metadata.location }}</value>
+<key>Region</key>: <value>{{ ip_context.metadata.region }}</value>
 <key>Organization</key>: <value>{{ ip_context.metadata.organization }}</value>
 <key>OS</key>: <value>{{ ip_context.metadata.os }}</value>
 <key>rDNS</key>: <value>{{ ip_context.metadata.rdns }}</value>
+<key>Spoofable</key>: <value>{{ ip_context.metadata.spoofable|string }}</value>
 <key>Tor</key>: <value>{{ ip_context.metadata.tor }}</value>
 
           <header>RAW DATA</header>
 ----------------------------
+{%- if ip_context.cve %}
+[CVE]
+{%- call(cve) macros.verbose_list(ip_context.cve) -%}
+- <key>CVE</key>: <value>{{ cve }}</value>
+{% endcall -%}
+{% endif %}
+
 {%- if ip_context.raw_data.scan %}
 [Scan]
 {%- call(scan) macros.verbose_list(ip_context.raw_data.scan) -%}
@@ -39,6 +48,13 @@
 {% endcall -%}
 {% endif %}
 
+{%- if ip_context.raw_data.web.useragents %}
+[Useragents]
+{%- call(useragent) macros.verbose_list(ip_context.raw_data.web.useragents) -%}
+- <value>{{ useragent }}</value>
+{% endcall -%}
+{% endif %}
+
 {%- if ip_context.raw_data.ja3 %}
 [JA3]
 {%- call(ja3) macros.verbose_list(ip_context.raw_data.ja3) -%}

diff --git a/src/greynoise/cli/templates/ip_context_result_query.txt.j2 b/src/greynoise/cli/templates/ip_context_result_query.txt.j2
@@ -0,0 +1,68 @@
+{% import "macros.txt.j2" as macros with context %}
+{%- if ip_context.seen %}
+          <header>OVERVIEW</header>
+----------------------------
+<key>Actor</key>: <value>{{ ip_context.actor }}</value>
+<key>Classification</key>: {{ macros.classification(ip_context.classification) }}
+<key>First seen</key>: <value>{{ ip_context.first_seen }}</value>
+<key>IP</key>: <value>{{ ip_context.ip }}</value>
+<key>Last seen</key>: <value>{{ ip_context.last_seen }}</value>
+{% if ip_context.tags -%}
+<key>Tags</key>:
+{%- call(tag) macros.verbose_list(ip_context.tags) -%}
+- <value>{{ tag }}</value>
+{% endcall -%}
+{% endif %}
+          <header>METADATA</header>
+----------------------------
+<key>ASN</key>: <value>{{ ip_context.metadata.asn }}</value>
+<key>Category</key>: <value>{{ ip_context.metadata.category }}</value>
+<key>Location</key>: <value>{{ ip_context.metadata.location }}</value>
+<key>Region</key>: <value>{{ ip_context.metadata.region }}</value>
+<key>Organization</key>: <value>{{ ip_context.metadata.organization }}</value>
+<key>OS</key>: <value>{{ ip_context.metadata.os }}</value>
+<key>rDNS</key>: <value>{{ ip_context.metadata.rdns }}</value>
+<key>Spoofable</key>: <value>{{ ip_context.spoofable|string }}</value>
+<key>Tor</key>: <value>{{ ip_context.metadata.tor }}</value>
+
+          <header>RAW DATA</header>
+----------------------------
+{%- if ip_context.cve %}
+[CVE]
+{%- call(cve) macros.verbose_list(ip_context.cve) -%}
+- <key>CVE</key>: <value>{{ cve }}</value>
+{% endcall -%}
+{% endif %}
+
+{%- if ip_context.raw_data.scan %}
+[Scan]
+{%- call(scan) macros.verbose_list(ip_context.raw_data.scan) -%}
+- <key>Port/Proto</key>: <value>{{ scan.port }}/{{ scan.protocol }}</value>
+{% endcall -%}
+{% endif %}
+
+{%- if ip_context.raw_data.web.paths %}
+[Paths]
+{%- call(path) macros.verbose_list(ip_context.raw_data.web.paths) -%}
+- <value>{{ path }}</value>
+{% endcall -%}
+{% endif %}
+
+{%- if ip_context.raw_data.web.useragents %}
+[Useragents]
+{%- call(useragent) macros.verbose_list(ip_context.raw_data.web.useragents) -%}
+- <value>{{ useragent }}</value>
+{% endcall -%}
+{% endif %}
+
+{%- if ip_context.raw_data.ja3 %}
+[JA3]
+{%- call(ja3) macros.verbose_list(ip_context.raw_data.ja3) -%}
+- <key>Port</key>: <value>{{ ja3.port }}</value>, <key>Fingerprint</key>: <value>{{ ja3.fingerprint }}</value>
+{% endcall -%}
+{% endif %}
+{%- elif ip_context.error %}
+{{ ip_context.error }}
+{% else %}
+{{ ip_context.ip }} has not been seen in scans in the past 30 days.
+{% endif %}
diff --git a/src/greynoise/cli/templates/macros.txt.j2 b/src/greynoise/cli/templates/macros.txt.j2
@@ -42,6 +42,20 @@ Showing results 1 - {{ max_elements }}. Run again with -v for full output.
 {% endif -%}
 {% endmacro %}
 
+# Render tuples of element names and counts properly aligned
+# Width is based on the longest element in the list for each column
+# Also render follow same rules as "verbose_list" to render long lists
+{% macro column_verbose_list_bool(elements, field_name) %}
+{%- set max_elements = 20 %}
+{%- set elements_slice = elements[:max_elements if verbose < 1 else None] %}
+{%- set right_width = [6] | max %}
+{%- set left_width_verbose = [1] | max %}
+{%- set left_width = left_width_verbose if verbose > 1 else [left_width_verbose, max_width - 3 - right_width] | min %}
+{%- for element in elements_slice %}
+- <key>{{ "%-*s" | format(left_width, element[field_name]) }}</key> <value>{{ "%*s" | format(right_width, element.count) }}</value>
+{%- endfor %}
+{% endmacro %}
+
 # Render classification with proper color
 {% macro classification(value) %}
 {%- if value == "benign" -%}

diff --git a/src/greynoise/cli/templates/stats.txt.j2 b/src/greynoise/cli/templates/stats.txt.j2
@@ -34,6 +34,11 @@
 {{- macros.column_verbose_list(result.stats.organizations, "organization") }}
 {%- endif %}
 
+{%- if result.stats.spoofable %}
+<header>Spoofable</header>:
+{{- macros.column_verbose_list_bool(result.stats.spoofable, "spoofable") }}
+{%- endif %}
+
 {%- if result.stats.tags %}
 <header>Tags</header>:
 {{- macros.column_verbose_list(result.stats.tags, "tag") }}

diff --git a/src/greynoise/util.py b/src/greynoise/util.py
@@ -149,8 +149,10 @@ def validate_ip(ip_address, strict=True):
 
     """
 
-    valid_ip_regex = r"^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]| \
-                        2[0-4][0-9]|[01]?[0-9][0-9]?)$"  # noqa
+    valid_ip_regex = (
+        r"^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|"
+        r"2[0-4][0-9]|[01]?[0-9][0-9]?)$"
+    )
     if re.match(valid_ip_regex, ip_address):
         return True
     else:

diff --git a/tests/cli/test_formatter.py b/tests/cli/test_formatter.py
@@ -18,6 +18,7 @@
 EXAMPLE_IP_CONTEXT = {
     "actor": "<actor>",
     "classification": "<classification>",
+    "cve": ["<cve#1>", "<cve#2>"],
     "first_seen": "<first_seen>",
     "ip": "<ip_address>",
     "last_seen": "<last_seen>",
@@ -29,8 +30,12 @@
         "country_code": "<country_code>",
         "organization": "<organization>",
         "os": "<os>",
+        "region": "<region>",
         "rdns": "<rdns>",
         "tor": False,
+        "spoofable": False,
+        "vpn": False,
+        "vpn_service": "<vpn_service>",
     },
     "raw_data": {
         "ja3": [
@@ -48,6 +53,7 @@
         },
     },
     "seen": True,
+    "spoofable": False,
     "tags": ["<tag#1>", "<tag#2>", "<tag#3>"],
 }
 
@@ -71,13 +77,19 @@
         <key>ASN</key>: <value><asn></value>
         <key>Category</key>: <value><category></value>
         <key>Location</key>: <value><city>, <country> (<country_code>)</value>
+        <key>Region</key>: <value><region></value>
         <key>Organization</key>: <value><organization></value>
         <key>OS</key>: <value><os></value>
         <key>rDNS</key>: <value><rdns></value>
+        <key>Spoofable</key>: <value>False</value>
         <key>Tor</key>: <value>False</value>
 
                   <header>RAW DATA</header>
         ----------------------------
+        [CVE]
+        - <key>CVE</key>: <value><cve#1></value>
+        - <key>CVE</key>: <value><cve#2></value>
+
         [Scan]
         - <key>Port/Proto</key>: <value>123456/TCP</value>
         - <key>Port/Proto</key>: <value>123456/UDP</value>
@@ -87,6 +99,11 @@
         - <value>/favicon.ico</value>
         - <value>/robots.txt</value>
 
+        [Useragents]
+        - <value><useragent#1></value>
+        - <value><useragent#2></value>
+        - <value><useragent#3></value>
+
         [JA3]
         - <key>Port</key>: <value>123456</value>, <key>Fingerprint</key>: <value><fingerprint#1></value>
         - <key>Port</key>: <value>123456</value>, <key>Fingerprint</key>: <value><fingerprint#2></value>

diff --git a/tests/test_api.py b/tests/test_api.py
@@ -661,3 +661,16 @@ def test_metadata(self, client):
         response = client.metadata()
         client._request.assert_called_with("meta/metadata")
         assert response == expected_response
+
+
+class TestMetaPing(object):
+    """GreyNoise client run Meta Ping test cases."""
+
+    def test_metadata_ping(self, client):
+        """Run GNQL stats query."""
+        expected_response = "Success: Access and API Key Valid"
+
+        client._request = Mock(return_value=expected_response)
+        response = client.test_connection()
+        client._request.assert_called_with("meta/ping")
+        assert response == expected_response