From 21ef5e46352576065a1ce85d938a41d01831c156 Mon Sep 17 00:00:00 2001 From: Brad Chiappetta <38439955+bradchiappetta@users.noreply.github.com> Date: Wed, 9 Feb 2022 14:02:30 -0500 Subject: [PATCH] v1.2.0 Updates (#534) * API client: * Add ip_multi command to support bulk IP Context lookups * Fix issue with "include_invalid" option on quick command failing with "riot" key missing * CLI: * Add ip-multi command to support bulk IP Context lookups * Add support for comma separated IP lists for ip-multi and quick commands * Add size and scroll arguments for query and stats command * Update quick command to not error completely when non-routable IP is passed as an input in a list * Dependencies: * Added colorama dependency * Update Click to 8.0.3 * Updated cachetools to 5.0.0 * Updated jinja to 3.0.3 * Updated more-itertools to 8.12.0 * Updated requests to 2.27.1 * Updated structlog to 21.5.0 --- .bumpversion.cfg | 2 +- .github/CODEOWNERS | 3 + .gitignore | 2 + CHANGELOG.rst | 27 +++ docs/source/conf.py | 2 +- docs/source/index.rst | 2 +- docs/source/introduction.rst | 4 +- docs/source/tutorial.rst | 132 ++++++++++-- requirements/common.txt | 13 +- requirements/dev.txt | 6 +- requirements/docs.txt | 6 +- requirements/test.txt | 18 +- setup.cfg | 1 + setup.py | 2 +- src/greynoise/__version__.py | 2 +- src/greynoise/api/__init__.py | 159 ++++++++++++--- src/greynoise/cli/decorator.py | 2 + src/greynoise/cli/formatter.py | 8 + src/greynoise/cli/parameter.py | 12 +- src/greynoise/cli/subcommand.py | 32 ++- src/greynoise/cli/templates/gnql_query.txt.j2 | 1 + .../cli/templates/ip_multi_context.txt.j2 | 70 +++++++ src/greynoise/util.py | 14 +- tests/cli/test_formatter.py | 12 +- tests/cli/test_subcommand.py | 193 +++++++++++++++++- 25 files changed, 641 insertions(+), 84 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 src/greynoise/cli/templates/ip_multi_context.txt.j2 diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cf1dfa08..dae31810 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.1.0 +current_version = 1.2.0 tag = False commit = False diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..ac10dee2 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# Github codeowners file to improve our lives when selecting users for PR's + +* @bradchiappetta @elliottminns @Obsecurus @superducktoes @nathanqthai diff --git a/.gitignore b/.gitignore index a0805751..98872a72 100644 --- a/.gitignore +++ b/.gitignore @@ -90,6 +90,8 @@ venv/ venv_py2/ venv3/ ENV/ +bin/ +pyvenv.cfg # Spyder project settings .spyderproject diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dd52b1e0..6c44bb2f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,32 @@ Changelog ========= +Version `1.2.0`_ +================ +**Date**: September 03, 2021 + +* API client: + + * Add ip_multi command to support bulk IP Context lookups + * Fix issue with "include_invalid" option on quick command failing with "riot" key missing + +* CLI: + + * Add ip-multi command to support bulk IP Context lookups + * Add support for comma separated IP lists for ip-multi and quick commands + * Add size and scroll arguments for query and stats command + * Update quick command to not error completely when non-routable IP is passed as an input in a list + +* Dependencies: + + * Added colorama dependency + * Update Click to 8.0.3 + * Updated cachetools to 5.0.0 + * Updated jinja to 3.0.3 + * Updated more-itertools to 8.12.0 + * Updated requests to 2.27.1 + * Updated structlog to 21.5.0 + Version `1.1.0`_ ================ **Date**: June 23, 2021 @@ -238,3 +264,4 @@ Version `0.2.0`_ .. _`0.9.1`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v0.9.0...0.9.1 .. _`1.0.0`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v0.9.1...1.0.0 .. _`1.1.0`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v1.0.0...1.1.0 +.. _`1.2.0`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v1.1.0...1.2.0 diff --git a/docs/source/conf.py b/docs/source/conf.py index 32125190..71ba2ee3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -25,7 +25,7 @@ author = "GreyNoise Intelligence" # The full version, including alpha/beta/rc tags -release = "1.1.0" +release = "1.2.0" # -- General configuration --------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index b1ad1025..0465c5ee 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -12,7 +12,7 @@ Python GreyNoise License ======= -Copyright 2018-2021 GreyNoise Intelligence +Copyright 2018-2022 GreyNoise Intelligence Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index 2f226750..23bf949c 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -15,4 +15,6 @@ In particular, it will allow you to: - get a list of noise IP address found in a given date. -.. _GreyNoise API: https://developer.greynoise.io/reference +- check if an IP address belongs to a common business service + +.. _GreyNoise API: https://docs.greynoise.io/reference diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 0c17e5c2..6052ba67 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -136,21 +136,89 @@ Detailed context information for any given IP address is also available:: } } +When there's a list of IP addresses to get full context from, they can be checked all at once like +this (this method also supports the include_invalid flag:: + + >>> api_client.ip_multi(['8.8.8.8', '58.220.219.247']) + [ + { + 'ip': '8.8.8.8', + 'first_seen': '', + 'last_seen': '', + 'seen': False, + 'tags': None, + 'actor': '', + 'spoofable': False, + 'classification': '', + 'cve': None, + 'bot': False, + 'vpn': False, + 'vpn_service': '', + 'metadata': { + 'asn': '', + 'city': '', + 'country': '', + 'country_code': '', + 'organization': '', + 'category': '', + 'tor': False, + 'rdns': '', + 'os': '' + }, + 'raw_data': { + 'scan': [], + 'web': {}, + 'ja3': [], + 'hassh': [] + } + }, + { + 'ip': '58.220.219.247', + 'first_seen': '', + 'last_seen': '', + 'seen': False, + 'tags': None, + 'actor': '', + 'spoofable': False, + 'classification': '', + 'cve': None, + 'bot': False, + 'vpn': False, + 'vpn_service': '', + 'metadata': { + 'asn': '', + 'city': '', + 'country': '', + 'country_code': '', + 'organization': '', + 'category': '', + 'tor': False, + 'rdns': '', + 'os': '' + }, + 'raw_data': { + 'scan': [], + 'web': {}, + 'ja3': [], + 'hassh': [] + } + } + ] + Any IP can also be checked to see if it exists within the RIOT dataset:: - >>> api_client.riot('58.220.219.247') + >>> api_client.riot('8.8.8.8') { - 'ip': '8.8.8.8', - 'riot': True, - 'category': 'public_dns', - 'name': 'Google Public DNS', - 'description': "Google's global domain name system (DNS) resolution service.", - 'explanation': "Public DNS services are used as alternatives to ISP's name servers. You may - see devices on your network communicating with Google Public DNS over port 53/TCP or 53/UDP - to resolve DNS lookups.", - 'last_updated': '2021-01-06T01:56:45Z', - 'logo_url': 'https://www.gstatic.com/devrel-devsite/prod/v9d82702993bc22f782b7874a0f933b5e39c1f0889acab7d1fce0d6deb8e0f63d/cloud/images/cloud-logo.svg', - 'reference': 'https://developers.google.com/speed/public-dns/docs/isp#alternative' + 'ip': '8.8.8.8', + 'riot': True, + 'category': 'public_dns', + 'name': 'Google Public DNS', + 'description': "Google's global domain name system (DNS) resolution service.", + 'explanation': "Public DNS services are used as alternatives to ISP's name servers. You may see devices on your network communicating with Google Public DNS over port 53/TCP or 53/UDP to resolve DNS lookups.", + 'last_updated': '2022-02-08T18:58:27Z', + 'logo_url': 'https://upload.wikimedia.org/wikipedia/commons/2/2f/Google_2015_logo.svg', + 'reference': 'https://developers.google.com/speed/public-dns/docs/isp#alternative', + 'trust_level': '1' } .. note:: @@ -505,7 +573,7 @@ Internet as follows:: 58.220.219.247 is classified as NOISE. When there's a list of IP addresses to verify, they can be checked all at once like -this:: +this (a comma seperated list is also supported:: $ greynoise quick 8.8.8.8 58.220.219.247 8.8.8.8 is classified as NOT NOISE. @@ -548,6 +616,44 @@ Detailed context information for any given IP address is also available:: - Port/Proto: 3389/TCP - Port/Proto: 65529/TCP +When there's a list of IP addresses to verify, they can be checked all at once like +this (a comma seperated list is also supported:: + + $ greynoise ip-multi 8.8.8.8 58.220.219.247 + OVERVIEW + ---------------------------- + Actor: unknown + Classification: malicious + First seen: 2020-12-21 + IP: 42.230.170.174 + Last seen: 2022-02-08 + Tags: + - Mirai + + METADATA + ---------------------------- + ASN: AS4837 + Category: isp + Location: + Region: Heilongjiang + Organization: CHINA UNICOM China169 Backbone + OS: Linux 2.2-3.x + rDNS: hn.kd.ny.adsl + Spoofable: False + Tor: False + + RAW DATA + ---------------------------- + [Scan] + - Port/Proto: 23/TCP + - Port/Proto: 8080/TCP + + [Paths] + - /setup.cgi + + 8.8.8.8 is classified as NOT NOISE. + + GNQL ---- diff --git a/requirements/common.txt b/requirements/common.txt index bfc3d194..d314fc0d 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -1,14 +1,15 @@ -Click==8.0.1 +Click==8.0.3 ansimarkup==1.4.0 -cachetools==4.2.2;python_version>='3' +cachetools==5.0.0;python_version>='3' +colorama==0.4.4 click-default-group==1.2.2 click-repl==0.2.0 dict2xml==1.7.0;python_version>='3' ipaddress==1.0.23 jinja2==2.11.3;python_version=='3.5' # pyup: ignore -jinja2==3.0.1;python_version>='3.6' -more-itertools==8.8.0;python_version>='3' -requests==2.25.1 +jinja2==3.0.3;python_version>='3.6' +more-itertools==8.12.0;python_version>='3' +requests==2.27.1 six==1.16.0 structlog==20.1.0;python_version=='3.5' # pyup: ignore -structlog==21.1.0;python_version>='3.6' +structlog==21.5.0;python_version>='3.6' diff --git a/requirements/dev.txt b/requirements/dev.txt index d0f91b50..78586882 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,6 +1,6 @@ # Requirements needed to develop the application -r test.txt advbumpversion==1.2.0 -ipython==7.25.0;python_version>='3' -pre-commit==2.13.0 -tox==3.23.1 +ipython==8.0.1;python_version>='3' +pre-commit==2.17.0 +tox==3.24.5 diff --git a/requirements/docs.txt b/requirements/docs.txt index 18131228..6231683f 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,5 @@ # Requirements needed to build the documentation -r common.txt -Sphinx==4.0.2 -sphinx-click==3.0.1 -sphinx-rtd-theme==0.5.2 +Sphinx==4.4.0 +sphinx-click==3.0.3 +sphinx-rtd-theme==1.0.0 diff --git a/requirements/test.txt b/requirements/test.txt index 84d8bd15..6677ba0c 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,14 +1,16 @@ -black==21.6b0;python_version>='3.6' -flake8==3.9.2 +black==22.1.0;python_version>='3.6' +flake8==3.9.2;python_version=='3.5' # pyup: ignore +flake8==4.0.1;python_version>='3.6' isort==4.3.21;python_version=='3.5' # pyup: ignore -isort==5.9.1;python_version>='3.6' +isort==5.10.1;python_version>='3.6' mock==3.0.5;python_version=='3.5' # pyup: ignore mock==4.0.3;python_version>='3.6' pylint==2.6.2;python_version=='3.5' # pyup: ignore -pylint==2.9.0;python_version>='3.6' -pytest-cov==2.12.1 +pylint==2.12.2;python_version>='3.6' +pytest-cov==2.12.1;python_version=='3.5' # pyup: ignore +pytest-cov==3.0.0;python_version>='3.6' pytest==6.1.2;python_version=='3.5' # pyup: ignore -pytest==6.2.4;python_version>='3.6' +pytest==6.2.5;python_version>='3.6' restructuredtext-lint==1.3.2 -twine==3.4.1;python_version>='3.6' -yamllint==1.26.1;python_version>='3' +twine==3.7.1;python_version>='3.6' +yamllint==1.26.3;python_version>='3' diff --git a/setup.cfg b/setup.cfg index af29475f..bd078878 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,7 @@ [flake8] max-line-length = 88 max-complexity = 10 +ignore = C901,W503 [isort] multi_line_output = 3 diff --git a/setup.py b/setup.py index e2836ea7..5ad0667b 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ def read(fname): setup( name="greynoise", - version="1.1.0", + version="1.2.0", description="Abstraction to interact with GreyNoise API.", url="https://greynoise.io/", author="GreyNoise Intelligence", diff --git a/src/greynoise/__version__.py b/src/greynoise/__version__.py index 17b1462a..4467612f 100644 --- a/src/greynoise/__version__.py +++ b/src/greynoise/__version__.py @@ -5,4 +5,4 @@ __maintainer__ = "GreyNoise Intelligence" __email__ = "hello@greynoise.io" __status__ = "BETA" -__version__ = "1.1.0" +__version__ = "1.2.0" diff --git a/src/greynoise/api/__init__.py b/src/greynoise/api/__init__.py index 3d1d568c..1b2db61e 100644 --- a/src/greynoise/api/__init__.py +++ b/src/greynoise/api/__init__.py @@ -25,7 +25,7 @@ def initialize_cache(cache_max_size, cache_ttl): return cache -class GreyNoise(object): +class GreyNoise(object): # pylint: disable=R0205,R0902 """GreyNoise API client. @@ -45,6 +45,7 @@ class GreyNoise(object): EP_INTERESTING = "interesting/{ip_address}" EP_NOISE_MULTI = "noise/multi/quick" EP_NOISE_CONTEXT = "noise/context/{ip_address}" + EP_NOISE_CONTEXT_MULTI = "noise/multi/context" EP_COMMUNITY_IP = "v3/community/{ip_address}" EP_META_METADATA = "meta/metadata" EP_PING = "ping" @@ -97,7 +98,7 @@ def __init__( cache_max_size=None, cache_ttl=None, offering=None, - ): + ): # pylint: disable=R0913 if any( configuration_value is None for configuration_value in (api_key, timeout, api_server, proxy, offering) @@ -153,9 +154,11 @@ def _request(self, endpoint, params=None, json=None, method="get"): if params is None: params = {} - user_agent_parts = ["GreyNoise/{}".format(__version__)] + user_agent_parts = ["GreyNoise/{}".format(__version__)] # pylint: disable=C0209 if self.integration_name: - user_agent_parts.append("({})".format(self.integration_name)) + user_agent_parts.append( + "({})".format(self.integration_name) + ) # pylint: disable=C0209 headers = { "User-Agent": " ".join(user_agent_parts), "key": self.api_key, @@ -224,10 +227,11 @@ def analyze(self, text): response = [ {"message": "Quick Lookup not supported with Community offering"} ] - return response else: analyzer = Analyzer(self) - return analyzer.analyze(text) + response = analyzer.analyze(text) + + return response def filter(self, text, noise_only=False, riot_only=False): """Filter lines that contain IP addresses from a given text. @@ -246,8 +250,8 @@ def filter(self, text, noise_only=False, riot_only=False): :rtype: iterable """ - filter = Filter(self) - for filtered_chunk in filter.filter( + gnfilter = Filter(self) + for filtered_chunk in gnfilter.filter( text, noise_only=noise_only, riot_only=riot_only ): yield filtered_chunk @@ -263,7 +267,6 @@ def interesting(self, ip_address): response = { "message": "Interesting report not supported with Community offering" } - return response else: LOGGER.debug( "Reporting interesting IP: %s...", ip_address, ip_address=ip_address @@ -272,9 +275,10 @@ def interesting(self, ip_address): endpoint = self.EP_INTERESTING.format(ip_address=ip_address) response = self._request(endpoint, method="post") - return response - def ip(self, ip_address): + return response + + def ip(self, ip_address): # pylint: disable=C0103 """Get context associated with an IP address. :param ip_address: IP address to use in the look-up. @@ -320,7 +324,6 @@ def query(self, query, size=None, scroll=None): """Run GNQL query.""" if self.offering == "community": response = {"message": "GNQL not supported with Community offering"} - return response else: LOGGER.debug( "Running GNQL query: %s...", @@ -335,9 +338,10 @@ def query(self, query, size=None, scroll=None): if scroll is not None: params["scroll"] = scroll response = self._request(self.EP_GNQL, params=params) - return response - def quick(self, ip_addresses, include_invalid=False): # noqa: C901 + return response + + def quick(self, ip_addresses, include_invalid=False): # pylint: disable=R0912,R0914 """Get activity associated with one or more IP addresses. :param ip_addresses: One or more IP addresses to use in the look-up. @@ -353,7 +357,6 @@ def quick(self, ip_addresses, include_invalid=False): # noqa: C901 response = [ {"message": "Quick Lookup not supported with Community offering"} ] - return response else: if isinstance(ip_addresses, str): ip_addresses = ip_addresses.split(",") @@ -363,7 +366,7 @@ def quick(self, ip_addresses, include_invalid=False): # noqa: C901 valid_ip_addresses = [ ip_address for ip_address in ip_addresses - if validate_ip(ip_address, strict=False) + if validate_ip(ip_address, strict=False, print_warning=False) ] if self.use_cache: @@ -411,31 +414,132 @@ def quick(self, ip_addresses, include_invalid=False): # noqa: C901 else: results.append(result) - [ - results.append({"ip": ip, "noise": False, "code": "404"}) - for ip in ip_addresses - if ip not in valid_ip_addresses and include_invalid - ] + if include_invalid: + for ip_address in ip_addresses: + if ip_address not in valid_ip_addresses: + results.append( + { + "ip": ip_address, + "noise": False, + "riot": False, + "code": "404", + } + ) for result in results: code = result["code"] result["code_message"] = self.CODE_MESSAGES.get( code, self.UNKNOWN_CODE_MESSAGE.format(code) ) - return results + response = results + + return response + + def ip_multi(self, ip_addresses, include_invalid=False): # pylint: disable=R0912 + """Get activity associated with one or more IP addresses. + + :param ip_addresses: One or more IP addresses to use in the look-up. + :type ip_addresses: str | list + :return: Bulk status information for IP addresses. + :rtype: dict + + :param include_invalid: True or False + :type include_invalid: bool + + """ + if self.offering == "community": # pylint: disable=R1702 + results = [ + {"message": "IP Multi Lookup not supported with Community offering"} + ] + else: + if isinstance(ip_addresses, str): + ip_addresses = ip_addresses.split(",") + + LOGGER.debug("Getting noise context...", ip_addresses=ip_addresses) + + valid_ip_addresses = [ + ip_address + for ip_address in ip_addresses + if validate_ip(ip_address, strict=False, print_warning=False) + ] + + if self.use_cache: + cache = self.ip_context_cache + # Keep the same ordering as in the input + ordered_results = OrderedDict( + (ip_address, cache.get(ip_address)) + for ip_address in valid_ip_addresses + ) + api_ip_addresses = [ + ip_address + for ip_address, result in ordered_results.items() + if result is None + ] + if api_ip_addresses: + api_results = [] + chunks = more_itertools.chunked( + api_ip_addresses, self.IP_QUICK_CHECK_CHUNK_SIZE + ) + for chunk in chunks: + api_result = self._request( + self.EP_NOISE_CONTEXT_MULTI, + method="post", + json={"ips": chunk}, + ) + + api_result = api_result["data"] + + if isinstance(api_result, list): + api_results.extend(api_result) + else: + api_results.append(api_result) + + for ip_address in valid_ip_addresses: + if ip_address not in api_results: + api_results.append({"ip": ip_address, "seen": False}) + + for result in api_results: + ip_address = result["ip"] + + ordered_results[ip_address] = cache.setdefault( + ip_address, result + ) + + results = list(ordered_results.values()) + + else: + results = [] + chunks = more_itertools.chunked( + valid_ip_addresses, self.IP_QUICK_CHECK_CHUNK_SIZE + ) + for chunk in chunks: + result = self._request( + self.EP_NOISE_CONTEXT_MULTI, json={"ips": chunk} + ) + if isinstance(result, list): + results.extend(result) + else: + results.append(result) + + if include_invalid: + for ip_address in ip_addresses: + if ip_address not in valid_ip_addresses: + results.append({"ip": ip_address, "seen": False}) + + return results def stats(self, query, count=None): """Run GNQL stats query.""" if self.offering == "community": response = {"message": "Stats Query not supported with Community offering"} - return response else: LOGGER.debug("Running GNQL stats query: %s...", query, query=query) params = {"query": query} if count is not None: params["count"] = count response = self._request(self.EP_GNQL_STATS, params=params) - return response + + return response def metadata(self): """Get metadata.""" @@ -443,11 +547,11 @@ def metadata(self): response = { "message": "Metadata lookup not supported with Community offering" } - return response else: LOGGER.debug("Getting metadata...") response = self._request(self.EP_META_METADATA) - return response + + return response def test_connection(self): """Test the API connection and API key.""" @@ -466,7 +570,6 @@ def riot(self, ip_address): """ if self.offering == "community": response = {"message": "RIOT lookup not supported with Community offering"} - return response else: LOGGER.debug("Checking RIOT for %s...", ip_address, ip_address=ip_address) validate_ip(ip_address) @@ -477,4 +580,4 @@ def riot(self, ip_address): if "ip" not in response: response["ip"] = ip_address - return response + return response diff --git a/src/greynoise/cli/decorator.py b/src/greynoise/cli/decorator.py index 880b828a..359633c1 100644 --- a/src/greynoise/cli/decorator.py +++ b/src/greynoise/cli/decorator.py @@ -136,6 +136,8 @@ def gnql_command(function): @click.command() @click.argument("query", required=False) + @click.option("--size", "size", help="Max number of results to return") + @click.option("--scroll", "scroll", help="Scroll token for pagination") @click.option("-k", "--api-key", help="Key to include in API requests") @click.option( "-O", diff --git a/src/greynoise/cli/formatter.py b/src/greynoise/cli/formatter.py index f08e3cf7..af66bbcb 100644 --- a/src/greynoise/cli/formatter.py +++ b/src/greynoise/cli/formatter.py @@ -117,6 +117,13 @@ def ip_quick_check_formatter(results, verbose): return template.render(results=results, verbose=verbose) +@colored_output +def ip_multi_context_formatter(results, verbose): + """Convert IP multi context result into human-readable text.""" + template = JINJA2_ENV.get_template("ip_multi_context.txt.j2") + return template.render(results=results, verbose=verbose) + + @colored_output def gnql_query_formatter(results, verbose): """Convert GNQL query result into human-readable text.""" @@ -174,5 +181,6 @@ def interesting_formatter(results, verbose): "stats": gnql_stats_formatter, "riot": riot_formatter, "interesting": interesting_formatter, + "ip-multi": ip_multi_context_formatter, }, } diff --git a/src/greynoise/cli/parameter.py b/src/greynoise/cli/parameter.py index 636997c1..2c786607 100644 --- a/src/greynoise/cli/parameter.py +++ b/src/greynoise/cli/parameter.py @@ -13,10 +13,18 @@ def ip_addresses_parameter(_context, _parameter, values): :raises click.BadParameter: when any IP address value is invalid """ + valid_ips = [] for value in values: try: - validate_ip(value) + if "," in value: + split_value = value.split(",") + for item in split_value: + validate_ip(item, strict=False) + valid_ips.append(item) + else: + validate_ip(value) + valid_ips.append(value) except ValueError: raise click.BadParameter(value) - return values + return valid_ips diff --git a/src/greynoise/cli/subcommand.py b/src/greynoise/cli/subcommand.py index 9baf254e..703406d2 100644 --- a/src/greynoise/cli/subcommand.py +++ b/src/greynoise/cli/subcommand.py @@ -195,11 +195,15 @@ def query( output_format, verbose, query, + size, + scroll, offering, ): """Run a GNQL (GreyNoise Query Language) query.""" queries = get_queries(context, input_file, query) - results = [api_client.query(query=item) for item in queries] + results = [ + api_client.query(query=item, size=size, scroll=scroll) for item in queries + ] return results @@ -222,6 +226,30 @@ def quick( return results +@ip_lookup_command +def ip_multi( + context, + api_client, + api_key, + input_file, + output_file, + output_format, + ip_address, + offering, +): + """ + Perform Context lookup for multiple IPs at once.\n + Example: greynoise ip-multi 1.1.1.1 2.2.2.2 3.3.3.3\n + Example: greynoise ip-multi 1.1.1.1,2.2.2.2,3.3.3.3\n + Example: greynoise ip-multi -i + """ + ip_addresses = get_ip_addresses(context, input_file, ip_address) + results = [] + if ip_addresses: + results.extend(api_client.ip_multi(ip_addresses=ip_addresses)) + return results + + @click.command() @click.option("-k", "--api-key", required=True, help="Key to include in API requests") @click.option( @@ -276,6 +304,8 @@ def stats( output_format, verbose, query, + size, + scroll, offering, ): """Get aggregate stats from a given GNQL query.""" diff --git a/src/greynoise/cli/templates/gnql_query.txt.j2 b/src/greynoise/cli/templates/gnql_query.txt.j2 index a7e3f3ea..0e48da0e 100644 --- a/src/greynoise/cli/templates/gnql_query.txt.j2 +++ b/src/greynoise/cli/templates/gnql_query.txt.j2 @@ -6,6 +6,7 @@ {% call macros.header(loop) %}Query{% endcall -%} Query: {{ result.query }} Count of IPs Returned: {{ result.count }} +Scroll Token: {{ result.scroll }} {% if result.count > 10 %} {{ space }} This output format is limited to 10 results, for all results use -f json for JSON output. diff --git a/src/greynoise/cli/templates/ip_multi_context.txt.j2 b/src/greynoise/cli/templates/ip_multi_context.txt.j2 new file mode 100644 index 00000000..155d37bd --- /dev/null +++ b/src/greynoise/cli/templates/ip_multi_context.txt.j2 @@ -0,0 +1,70 @@ +{% import "macros.txt.j2" as macros with context %} +{% for ip_multi_context in results %} +{%- if ip_multi_context.seen %} +
OVERVIEW
+---------------------------- +Actor: {{ ip_multi_context.actor }} +Classification: {{ macros.classification(ip_multi_context.classification) }} +First seen: {{ ip_multi_context.first_seen }} +IP: {{ ip_multi_context.ip }} +Last seen: {{ ip_multi_context.last_seen }} +{% if ip_multi_context.tags -%} +Tags: +{%- call(tag) macros.verbose_list(ip_multi_context.tags) -%} +- {{ tag }} +{% endcall -%} +{% endif %} +
METADATA
+---------------------------- +ASN: {{ ip_multi_context.metadata.asn }} +Category: {{ ip_multi_context.metadata.category }} +Location: {{ ip_multi_context.metadata.location }} +Region: {{ ip_multi_context.metadata.region }} +Organization: {{ ip_multi_context.metadata.organization }} +OS: {{ ip_multi_context.metadata.os }} +rDNS: {{ ip_multi_context.metadata.rdns }} +Spoofable: {{ ip_multi_context.spoofable|string }} +Tor: {{ ip_multi_context.metadata.tor }} + +
RAW DATA
+---------------------------- +{%- if ip_multi_context.cve %} +[CVE] +{%- call(cve) macros.verbose_list(ip_multi_context.cve) -%} +- CVE: {{ cve }} +{% endcall -%} +{% endif %} + +{%- if ip_multi_context.raw_data.scan %} +[Scan] +{%- call(scan) macros.verbose_list(ip_multi_context.raw_data.scan) -%} +- Port/Proto: {{ scan.port }}/{{ scan.protocol }} +{% endcall -%} +{% endif %} + +{%- if ip_multi_context.raw_data.web.paths %} +[Paths] +{%- call(path) macros.verbose_list(ip_multi_context.raw_data.web.paths) -%} +- {{ path }} +{% endcall -%} +{% endif %} + +{%- if ip_multi_context.raw_data.web.useragents %} +[Useragents] +{%- call(useragent) macros.verbose_list(ip_multi_context.raw_data.web.useragents) -%} +- {{ useragent }} +{% endcall -%} +{% endif %} + +{%- if ip_multi_context.raw_data.ja3 %} +[JA3] +{%- call(ja3) macros.verbose_list(ip_multi_context.raw_data.ja3) -%} +- Port: {{ ja3.port }}, Fingerprint: {{ ja3.fingerprint }} +{% endcall -%} +{% endif %} +{%- elif ip_multi_context.error %} +{{ ip_multi_context.error }} +{% else %} +{{ ip_multi_context.ip }} is classified as NOT NOISE. +{%- endif %} +{%- endfor %} diff --git a/src/greynoise/util.py b/src/greynoise/util.py index 41133421..5aeddd78 100644 --- a/src/greynoise/util.py +++ b/src/greynoise/util.py @@ -148,7 +148,7 @@ def save_config(config): config_parser.write(config_file) -def validate_ip(ip_address, strict=True): +def validate_ip(ip_address, strict=True, print_warning=True): """Check if the IPv4 address is valid. :param ip_address: IPv4 address value to validate. @@ -156,6 +156,8 @@ def validate_ip(ip_address, strict=True): :param strict: Whether to raise exception if validation fails. :type strict: bool :raises ValueError: When validation fails and strict is set to True. + :type print_warning: bool + :raises ValueError: By default, otherwise returns nothing """ is_valid = False @@ -164,8 +166,9 @@ def validate_ip(ip_address, strict=True): ipaddress.ip_address(ip_address) is_valid = True except ValueError: - error_message = "Invalid IP address: {!r}".format(ip_address) - LOGGER.warning(error_message, ip_address=ip_address) + if print_warning: + error_message = "Invalid IP address: {!r}".format(ip_address) + LOGGER.warning(error_message, ip_address=ip_address) if strict: raise ValueError(error_message) return False @@ -175,8 +178,9 @@ def validate_ip(ip_address, strict=True): if is_routable: return True else: - error_message = "Non-Routable IP address: {!r}".format(ip_address) - LOGGER.warning(error_message, ip_address=ip_address) + if print_warning: + error_message = "Non-Routable IP address: {!r}".format(ip_address) + LOGGER.warning(error_message, ip_address=ip_address) if strict: raise ValueError(error_message) return False diff --git a/tests/cli/test_formatter.py b/tests/cli/test_formatter.py index d0aaaf44..fbbf92d1 100644 --- a/tests/cli/test_formatter.py +++ b/tests/cli/test_formatter.py @@ -162,7 +162,7 @@ class TestIPContextFormatter(object): ], ANSI_MARKUP.parse( textwrap.dedent( - u"""\ + """\ ╔═══════════════════════════╗ ║
Context 1 of 3
║ ╚═══════════════════════════╝ @@ -174,7 +174,7 @@ class TestIPContextFormatter(object): + EXAMPLE_IP_CONTEXT_OUTPUT + ANSI_MARKUP.parse( textwrap.dedent( - u""" + """ ╔═══════════════════════════╗ @@ -238,6 +238,7 @@ class TestGNQLQueryFormatter(object): { "complete": True, "count": 1, + "scroll": "abcdefg", "data": [EXAMPLE_IP_CONTEXT], "message": "ok", "query": "", @@ -245,12 +246,13 @@ class TestGNQLQueryFormatter(object): ], ANSI_MARKUP.parse( textwrap.dedent( - u"""\ + """\ ╔═══════════════════════════╗ ║
Query 1 of 1
║ ╚═══════════════════════════╝ Query: Count of IPs Returned: 1 + Scroll Token: abcdefg ┌───────────────────────────┐ @@ -318,7 +320,7 @@ class TestGNQLStatsFormatter(object): ], ANSI_MARKUP.parse( textwrap.dedent( - u"""\ + """\ ╔═══════════════════════════╗ ║
Query 1 of 1
║ ╚═══════════════════════════╝ @@ -421,7 +423,7 @@ def test_format_riot(self, result, expected): ], ANSI_MARKUP.parse( textwrap.dedent( - u"""\ + """\ 0.0.0.0 is in RIOT dataset.
OVERVIEW
diff --git a/tests/cli/test_subcommand.py b/tests/cli/test_subcommand.py index 061d9926..bc041a87 100644 --- a/tests/cli/test_subcommand.py +++ b/tests/cli/test_subcommand.py @@ -81,7 +81,7 @@ class TestAnalyze(object): }, } DEFAULT_OUTPUT = textwrap.dedent( - u"""\ + """\ ╔═══════════════════════════╗ ║ Analyze ║ ╚═══════════════════════════╝ @@ -664,7 +664,7 @@ def test_query(self, api_client): result = runner.invoke(subcommand.query, ["-f", "json", query]) assert result.exit_code == 0 assert result.output.strip("\n") == expected - api_client.query.assert_called_with(query=query) + api_client.query.assert_called_with(query=query, size=None, scroll=None) def test_input_file(self, api_client): """Run query from input file.""" @@ -677,7 +677,7 @@ def test_input_file(self, api_client): result = runner.invoke(subcommand.query, ["-f", "json", "-i", StringIO(query)]) assert result.exit_code == 0 assert result.output.strip("\n") == expected - api_client.query.assert_called_with(query=query) + api_client.query.assert_called_with(query=query, size=None, scroll=None) def test_stdin_input(self, api_client): """Run query from stdin.""" @@ -690,7 +690,7 @@ def test_stdin_input(self, api_client): result = runner.invoke(subcommand.query, ["-f", "json"], input=query) assert result.exit_code == 0 assert result.output.strip("\n") == expected - api_client.query.assert_called_with(query=query) + api_client.query.assert_called_with(query=query, size=None, scroll=None) def test_no_query_passed(self, api_client): """Usage is returned if no query or input file is passed.""" @@ -933,6 +933,191 @@ def test_api_key_not_found(self): assert "Error: API key not found" in result.output +class TestIPMulti(object): + """IP-Multi subcommand tests.""" + + @pytest.mark.parametrize( + "ip_address, output_format, expected", + ( + ( + "8.8.8.8", + "json", + json.dumps( + [{"ip": "8.8.8.8", "noise": True}], indent=4, sort_keys=True + ), + ), + ( + "8.8.8.8", + "xml", + textwrap.dedent( + """\ + + + \t + \t\t8.8.8.8 + \t\tTrue + \t + """ + ), + ), + ( + "8.8.8.8", + "txt", + "8.8.8.8 is classified as NOT NOISE.", + ), + ), + ) + def test_ip_multi(self, api_client, ip_address, output_format, expected): + """Quickly check IP address.""" + runner = CliRunner() + + api_client.ip_multi.return_value = [ + OrderedDict((("ip", ip_address), ("noise", True))) + ] + + result = runner.invoke(subcommand.ip_multi, ["-f", output_format, ip_address]) + assert result.exit_code == 0 + assert result.output.strip("\n") == expected + api_client.ip_multi.assert_called_with(ip_addresses=[ip_address]) + + @pytest.mark.parametrize( + "ip_addresses, mock_response, expected", + ( + ( + ["8.8.8.8", "8.8.8.9"], + [ + OrderedDict([("ip", "8.8.8.8"), ("noise", True)]), + OrderedDict([("ip", "8.8.8.9"), ("noise", False)]), + ], + json.dumps( + [ + {"ip": "8.8.8.8", "noise": True}, + {"ip": "8.8.8.9", "noise": False}, + ], + indent=4, + sort_keys=True, + ), + ), + ), + ) + def test_input_file(self, api_client, ip_addresses, mock_response, expected): + """Quickly check IP address from input file.""" + runner = CliRunner() + + api_client.ip_multi.return_value = mock_response + + result = runner.invoke( + subcommand.ip_multi, ["-f", "json", "-i", StringIO("\n".join(ip_addresses))] + ) + assert result.exit_code == 0 + assert result.output.strip("\n") == expected + api_client.ip_multi.assert_called_with(ip_addresses=ip_addresses) + + @pytest.mark.parametrize( + "ip_addresses, mock_response, expected", + ( + ( + ["8.8.8.8", "8.8.8.9"], + [ + OrderedDict([("ip", "8.8.8.8"), ("noise", True)]), + OrderedDict([("ip", "8.8.8.9"), ("noise", False)]), + ], + json.dumps( + [ + {"ip": "8.8.8.8", "noise": True}, + {"ip": "8.8.8.9", "noise": False}, + ], + indent=4, + sort_keys=True, + ), + ), + ), + ) + def test_stdin_input(self, api_client, ip_addresses, mock_response, expected): + """Quickly check IP address from stdin.""" + runner = CliRunner() + + api_client.ip_multi.return_value = mock_response + + result = runner.invoke( + subcommand.ip_multi, ["-f", "json"], input="\n".join(ip_addresses) + ) + assert result.exit_code == 0 + assert result.output.strip("\n") == expected + api_client.ip_multi.assert_called_with(ip_addresses=ip_addresses) + + def test_no_ip_address_passed(self, api_client): + """Usage is returned if no IP address or input file is passed.""" + runner = CliRunner() + + with patch("greynoise.cli.helper.sys") as sys: + sys.stdin.isatty.return_value = True + result = runner.invoke( + subcommand.ip_multi, parent=Context(main, info_name="greynoise") + ) + assert result.exit_code == -1 + assert "Usage: greynoise ip-multi" in result.output + api_client.ip_multi.assert_not_called() + + def test_input_file_invalid_ip_addresses_passed(self, api_client): + """Error returned if only invalid IP addresses are passed in input file.""" + runner = CliRunner() + + expected = ( + "Error: at least one valid IP address must be passed either as an " + "argument (IP_ADDRESS) or through the -i/--input_file option." + ) + + result = runner.invoke( + subcommand.ip_multi, + ["-i", StringIO("not-an-ip")], + parent=Context(main, info_name="greynoise"), + ) + assert result.exit_code == -1 + assert "Usage: greynoise ip-multi" in result.output + assert expected in result.output + api_client.ip_multi.assert_not_called() + + def test_invalid_ip_address_as_argument(self, api_client): + """Quick subcommand fails when ip_address is invalid.""" + runner = CliRunner() + + expected = "Error: Invalid value for '[IP_ADDRESS]...': not-an-ip\n" + + result = runner.invoke(subcommand.ip_multi, ["not-an-ip"]) + assert result.exit_code == 2 + assert "Usage: ip-multi [OPTIONS] [IP_ADDRESS]..." in result.output + assert expected in result.output + api_client.ip_multi.assert_not_called() + + def test_request_failure(self, api_client): + """Error is displayed on API request failure.""" + runner = CliRunner() + + api_client.ip_multi.side_effect = RequestFailure( + 401, {"message": "forbidden", "status": "error"} + ) + expected = "API error: forbidden" + + result = runner.invoke(subcommand.ip_multi, ["8.8.8.8"]) + assert result.exit_code == -1 + assert expected in result.output + + def test_api_key_not_found(self): + """Error is displayed if API key is not found.""" + runner = CliRunner() + + with patch("greynoise.cli.decorator.load_config") as load_config: + load_config.return_value = {"api_key": ""} + result = runner.invoke( + subcommand.ip_multi, + ["8.8.8.8"], + parent=Context(main, info_name="greynoise"), + ) + assert result.exit_code == -1 + assert "Error: API key not found" in result.output + + class TestSignature(object): """Signature subcommand test cases."""