From c419d05eee0fc7386712a47b5b6b2e5ccfa7bc2a Mon Sep 17 00:00:00 2001 From: SB Date: Thu, 7 Dec 2023 15:54:19 +0800 Subject: [PATCH 01/33] Add --scrape and --probes list option to allow special use case to scrape the output for Prometheus --- ra-dns-check.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ra-dns-check.py b/ra-dns-check.py index 85adc3e..6026ace 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -188,6 +188,14 @@ 'default': 86400, 'help': 'The max age (seconds) of the RIPE Atlas probe info file (older than this and we download a new one). Default: 86400', 'type': 'integer'}, + 'scrape': { + 'default': '', + 'help': 'Scrape output for Prometheus', + 'type': 'string'}, + 'probes': { + 'default': [], + 'help': 'Selected probes list eg. "919,166,1049"', + 'type': 'list'}, 'exclusion_list_file': { 'default': None, 'help': 'Filename for probe ID exclusion list', @@ -260,6 +268,8 @@ parser.add_argument('-S', '--slow_threshold', help=options_sample_dict['slow_threshold']['help'], type=int, default=options_sample_dict['slow_threshold']['default']) parser.add_argument('-t', '--split_char', help=options_sample_dict['split_char']['help'], type=str, default=options_sample_dict['split_char']['default']) parser.add_argument('-u', '--print_summary_stats', help=options_sample_dict['print_summary_stats']['help'], action='store_true', default=options_sample_dict['print_summary_stats']['default']) +parser.add_argument('--scrape', help=options_sample_dict['scrape']['help'], action='store_true', default=options_sample_dict['scrape']['default']) +parser.add_argument('--probes', help=options_sample_dict['probes']['help'], type=str, default=options_sample_dict['probes']['default']) parser.add_argument('filename_or_msmid', help='one or two local filenames or RIPE Atlas Measurement IDs', nargs='+') parser.format_help() args = parser.parse_known_args() From 40c9db79ffcbeb3ec7da367012d855e5ed153734 Mon Sep 17 00:00:00 2001 From: SB <> Date: Thu, 7 Dec 2023 15:59:27 +0800 Subject: [PATCH 02/33] Refractor load_probe_properties by reducing it's complexity --- ra-dns-check.py | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 6026ace..ee515d1 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -568,7 +568,7 @@ class fmt: # Process the data, either from a local file or by requesting it over the # 'net from RIPE Atlas. # -def process_request(_data_source, _results_set_id, _unixtime): +def process_request(_data_source, _results_set_id, _unixtime, probes = []): logger.info('Trying to access data_source %s for unixtime %s\n' % (_data_source, _unixtime)) # First we try to open the _data_source as a local file. If it exists, # read in the measurement results from a filename the user has @@ -599,7 +599,8 @@ def process_request(_data_source, _results_set_id, _unixtime): # If we have no unixtime to request results from, then we get the latest results if _unixtime == 0: kwargs = { - "msm_id": measurement_id + "msm_id": measurement_id, + "probe_ids": probes } logger.info('Fetching latest results for Measurement %i from RIPE Atlas API...\n' % measurement_id) is_success, results = AtlasLatestRequest(**kwargs).create() @@ -613,7 +614,8 @@ def process_request(_data_source, _results_set_id, _unixtime): kwargs = { "msm_id": measurement_id, "start": _unixtime, - "stop": _stop_time + "stop": _stop_time, + "probe_ids": probes } logger.info('Fetching results for Measurement %i, start unixtime: %s stop unixtime: %s\n' % (measurement_id, _unixtime, _stop_time)) is_success, results = AtlasResultsRequest(**kwargs).create() @@ -746,7 +748,7 @@ def process_request(_data_source, _results_set_id, _unixtime): m_timestamps[_results_set_id].sort() m_seen_probe_ids[_results_set_id].sort() logger.debug('m_seen_probe_ids[_results_set_id] is %d\n' % len(m_seen_probe_ids[_results_set_id])) - return measurement_id + return measurement_id, results # END def process_request #################### @@ -913,11 +915,39 @@ def load_probe_properties(probe_ids, ppcf): ###### # # Data loading and summary stats reporting loop ... + +try: + probes = args[0].probes.split(',') +except: + probes = [] + +if args[0].scrape: + m, dnsresult = process_request(data_sources[results_set_id], results_set_id, unixtimes[results_set_id], probes) + p_probe_properties = load_probe_properties([dnsresult[x]['prb_id'] for x in range(len(dnsresult))], config['ripe_atlas_probe_properties_json_cache_file']) + for dnsprobe in dnsresult: + try: + probe_num = str(dnsprobe['prb_id']) + pop = dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0] + city = dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1] + ripe_atlas_latency = { 'measurement_id' : dnsprobe['msm_id'], + 'probe_id' : dnsprobe['prb_id'], + 'version' : dnsprobe['af'], + 'probe_asn' : p_probe_properties[probe_num]['asn_v4'], + 'probe_ip' : dnsprobe['from'], + 'country' : p_probe_properties[probe_num]['country_code'], + 'city' : city, + 'pop' : pop, + } + print ("ripe_atlas_latency{} {} {}".format(str(ripe_atlas_latency).replace(" ",""),dnsprobe['result']['rt'],dnsprobe['timestamp'])) + except: + pass + exit() + while results_set_id <= last_results_set_id: #for t in data_sources: # m will receive the measurement ID for the processed data source logger.debug('data_source: %s results_set_id: %i unixtime: %i\n' % (data_sources[results_set_id], results_set_id, unixtimes[results_set_id])) - m = process_request(data_sources[results_set_id], results_set_id, unixtimes[results_set_id]) + m = process_request(data_sources[results_set_id], results_set_id, unixtimes[results_set_id], probes) measurement_ids.append(m) ###### # Summary stats From f69de8f96511411ef06fb858f4de85eca4f396dd Mon Sep 17 00:00:00 2001 From: SB <> Date: Thu, 7 Dec 2023 16:00:19 +0800 Subject: [PATCH 03/33] Refractor load_probe_properties by reducing it's complexity --- ra-dns-check.py | 55 ++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index ee515d1..c2ea6c1 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -874,34 +874,33 @@ def load_probe_properties(probe_ids, ppcf): # Loop through the list of supplied (seen) probe ids and collect their # info/meta data from either our local file or the RIPE Atlas API logger.info ('Matching seen probes with probe data; will query RIPE Atlas API for probe info not in local cache...\n') - for p in probe_ids: - if p in all_probes_dict.keys(): - probe_cache_hits += 1 - logger.debug('Probe %s info found in local cache.' % p) - matched_probe_info[p] = all_probes_dict[p] - else: - # If it's not in the cache file, request it from RIPE - #logger.debug ('NOT cached, trying RIPE Atlas...') - try: - ripe_result = Probe(id=p) - # - matched_probe_info[p] = {'asn_v4': ripe_result.asn_v4, - 'asn_v6': ripe_result.asn_v6, - 'country_code': ripe_result.country_code, - 'address_v4': ripe_result.address_v4, - 'address_v6': ripe_result.address_v6} - probe_cache_misses += 1 - all_probes_dict[p] = matched_probe_info[p] - logger.debug('Probe %9s info fetched from RIPE' % p) - except: - # Otherwise, it's empty - # we did not find any information about the probe, so set values to '-' - matched_probe_info[p] = { 'asn_v4': '-', - 'asn_v6': '-', - 'country_code': '-', - 'address_v4': '-', - 'address_v6': '-' } - logger.debug('Failed to get info about probe ID %s in the local cache or from RIPE Atlas API.' % p) + # Using python set() data strucure instead of checking ccahe hits or misses + # python set() element is unique and using set theory, we can identify the differences + # and discover new element not found in the caches. This reduce computational complexity + dns_probes = set(str(x) for x in probe_ids) + all_probes = set(all_probes_dict.keys()) + new_probes = dns_probes.difference(all_probes) + for i in dns_probes: + matched_probe_info[i] = all_probes_dict[i] + try: + for p in new_probes: + ripe_result = Probe(id=p) + matched_probe_info[p] = {'asn_v4': ripe_result.asn_v4, + 'asn_v6': ripe_result.asn_v6, + 'country_code': ripe_result.country_code, + 'address_v4': ripe_result.address_v4, + 'address_v6': ripe_result.address_v6} + all_probes_dict[p] = matched_probe_info[p] + logger.debug('Probe %9s info fetched from RIPE' % p) + except: + # Otherwise, it's empty + # we did not find any information about the probe, so set values to '-' + matched_probe_info[p] = { 'asn_v4': '-', + 'asn_v6': '-', + 'country_code': '-', + 'address_v4': '-', + 'address_v6': '-' } + logger.debug('Failed to get info about probe ID %s in the local cache or from RIPE Atlas API.' % p) logger.info('cache hits: %i cache misses: %i.\n' % (probe_cache_hits, probe_cache_misses)) # Write out the local JSON cache file with open(ppcf, mode='w') as f: From 2e2dfe4010b2b3bda6c552a95373ca5d41cc82e5 Mon Sep 17 00:00:00 2001 From: SB <> Date: Thu, 7 Dec 2023 21:10:55 +0800 Subject: [PATCH 04/33] Update README.md on the new arguments and provide sample command --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 17fc97e..4805bcb 100644 --- a/README.md +++ b/README.md @@ -40,3 +40,12 @@ tidy up first, mainly to regenerate ra-dns-check.sh, like so: After the initial install of the venv or python dependencies, you probably don't need to do these again, unless the dependencies have changed. + +## Additional feature + +* Added a --scrape argument. This argument will use the time provided (dtime1) variable to obtain the + 'rt' of DNS query. The output are formatted to allow Prometheus to scrape. +* Additonally, another arugment --probes '' only output the specific probes values offered + by the +* ```ra-dns-check.py --datetime1 202210080000 12016241 --scrape``` +* ```ra-dns-check.py --datetime1 202210080000 12016241 --scrape --probes "999,99"``` From 72917bf5996b1cb7521681ac51f6e87f8130639f Mon Sep 17 00:00:00 2001 From: SB <> Date: Fri, 8 Dec 2023 10:43:40 +0800 Subject: [PATCH 05/33] Change pop->host and change City->pop labels for scrape output From eaa0c120073ce2d1ba2b5b05b4901f3b18891352 Mon Sep 17 00:00:00 2001 From: SB <> Date: Fri, 8 Dec 2023 10:53:30 +0800 Subject: [PATCH 06/33] Change pop->loc and print Prometheus scrape output header --- ra-dns-check.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index c2ea6c1..63b34a3 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -923,19 +923,22 @@ def load_probe_properties(probe_ids, ppcf): if args[0].scrape: m, dnsresult = process_request(data_sources[results_set_id], results_set_id, unixtimes[results_set_id], probes) p_probe_properties = load_probe_properties([dnsresult[x]['prb_id'] for x in range(len(dnsresult))], config['ripe_atlas_probe_properties_json_cache_file']) + print (''' +# HELP ripe_atlas_latency The number of milliseconds for response reported by this probe for the time period requested on this measurement +# TYPE ripe_atlas_latency gauge ''') for dnsprobe in dnsresult: try: probe_num = str(dnsprobe['prb_id']) - pop = dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0] - city = dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1] + host = dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0] + loc = dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1] ripe_atlas_latency = { 'measurement_id' : dnsprobe['msm_id'], 'probe_id' : dnsprobe['prb_id'], 'version' : dnsprobe['af'], 'probe_asn' : p_probe_properties[probe_num]['asn_v4'], 'probe_ip' : dnsprobe['from'], 'country' : p_probe_properties[probe_num]['country_code'], - 'city' : city, - 'pop' : pop, + 'loc' : loc, + 'host' : host, } print ("ripe_atlas_latency{} {} {}".format(str(ripe_atlas_latency).replace(" ",""),dnsprobe['result']['rt'],dnsprobe['timestamp'])) except: From 2448eea6806b7709dd7d8afd9ded18dc99923dc5 Mon Sep 17 00:00:00 2001 From: sbng Date: Fri, 8 Dec 2023 11:10:01 +0800 Subject: [PATCH 07/33] Add latitiude and Longtitude to the scrape output as part of the label --- ra-dns-check.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 63b34a3..8cfc02a 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -923,8 +923,7 @@ def load_probe_properties(probe_ids, ppcf): if args[0].scrape: m, dnsresult = process_request(data_sources[results_set_id], results_set_id, unixtimes[results_set_id], probes) p_probe_properties = load_probe_properties([dnsresult[x]['prb_id'] for x in range(len(dnsresult))], config['ripe_atlas_probe_properties_json_cache_file']) - print (''' -# HELP ripe_atlas_latency The number of milliseconds for response reported by this probe for the time period requested on this measurement + print ('''# HELP ripe_atlas_latency The number of milliseconds for response reported by this probe for the time period requested on this measurement # TYPE ripe_atlas_latency gauge ''') for dnsprobe in dnsresult: try: @@ -939,6 +938,8 @@ def load_probe_properties(probe_ids, ppcf): 'country' : p_probe_properties[probe_num]['country_code'], 'loc' : loc, 'host' : host, + 'lat' : p_probe_properties[probe_num]['latitude'], + 'long' : p_probe_properties[probe_num]['longitude'], } print ("ripe_atlas_latency{} {} {}".format(str(ripe_atlas_latency).replace(" ",""),dnsprobe['result']['rt'],dnsprobe['timestamp'])) except: From 73fc0c01b3d9131c41d121dec80bc76238dcc93d Mon Sep 17 00:00:00 2001 From: sbng Date: Fri, 8 Dec 2023 11:42:46 +0800 Subject: [PATCH 08/33] Fix issues 3 (change single quote to double) and 4 (change long to lon) From 5cd3eaa6aef926a53fca501725c02593c4ed48f9 Mon Sep 17 00:00:00 2001 From: sbng Date: Fri, 8 Dec 2023 11:49:20 +0800 Subject: [PATCH 09/33] Fix issues 3 (change single quote to double) and 4 (change long to lon) --- ra-dns-check.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 8cfc02a..78bdae1 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -939,11 +939,12 @@ def load_probe_properties(probe_ids, ppcf): 'loc' : loc, 'host' : host, 'lat' : p_probe_properties[probe_num]['latitude'], - 'long' : p_probe_properties[probe_num]['longitude'], + 'lon' : p_probe_properties[probe_num]['longitude'], } - print ("ripe_atlas_latency{} {} {}".format(str(ripe_atlas_latency).replace(" ",""),dnsprobe['result']['rt'],dnsprobe['timestamp'])) + print ("ripe_atlas_latency{} {} {}".format(str(ripe_atlas_latency).replace(" ","").replace('\'','\"'),dnsprobe['result']['rt'],dnsprobe['timestamp'])) except: pass +# breakpoint() exit() while results_set_id <= last_results_set_id: From a80e4056e72db984b6767b2babe421dd33ca0d81 Mon Sep 17 00:00:00 2001 From: sbng Date: Fri, 8 Dec 2023 19:59:01 +0800 Subject: [PATCH 10/33] Fix issues 3 - reformat dictionary ouput to Prometheus format for label declarations and quoting --- ra-dns-check.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 78bdae1..8cacd79 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -906,7 +906,14 @@ def load_probe_properties(probe_ids, ppcf): with open(ppcf, mode='w') as f: json.dump(all_probes_dict, f) return(matched_probe_info) - +#################### +# +# Input a dictionary -> Output a single line of scrape formatted string +def dict_string(d): + dict_str = "" + for i,v in d.items(): + dict_str += str(i) + "=\"" + str(v) + "\"," + return dict_str.rstrip(",") # END of all function defs ################################################## @@ -928,23 +935,23 @@ def load_probe_properties(probe_ids, ppcf): for dnsprobe in dnsresult: try: probe_num = str(dnsprobe['prb_id']) - host = dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0] - loc = dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1] - ripe_atlas_latency = { 'measurement_id' : dnsprobe['msm_id'], - 'probe_id' : dnsprobe['prb_id'], - 'version' : dnsprobe['af'], - 'probe_asn' : p_probe_properties[probe_num]['asn_v4'], - 'probe_ip' : dnsprobe['from'], - 'country' : p_probe_properties[probe_num]['country_code'], - 'loc' : loc, - 'host' : host, - 'lat' : p_probe_properties[probe_num]['latitude'], - 'lon' : p_probe_properties[probe_num]['longitude'], + delay = dnsprobe['result']['rt'] + timestamp = dnsprobe['timestamp'] + ripe_atlas_latency = { 'measurement_id' : str(dnsprobe['msm_id']), + 'probe_id' : str(dnsprobe['prb_id']), + 'version' : str(dnsprobe['af']), + 'probe_asn' : str(p_probe_properties[probe_num]['asn_v4']), + 'probe_ip' : str(dnsprobe['from']), + 'country' : str(p_probe_properties[probe_num]['country_code']), + 'loc' : str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1]), + 'host' : str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0]), + 'lat' : str(p_probe_properties[probe_num]['latitude']), + 'lon' : str(p_probe_properties[probe_num]['longitude']), } - print ("ripe_atlas_latency{} {} {}".format(str(ripe_atlas_latency).replace(" ","").replace('\'','\"'),dnsprobe['result']['rt'],dnsprobe['timestamp'])) + labels = dict_string(ripe_atlas_latency) + print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') except: pass -# breakpoint() exit() while results_set_id <= last_results_set_id: From 2da811e0c54bbe9ed5a45ac9ee0a91c3e845bc6e Mon Sep 17 00:00:00 2001 From: sbng Date: Sat, 9 Dec 2023 05:35:04 +0800 Subject: [PATCH 11/33] issues 4 - show ipv6 address and ASN if the probe is ipv6 capable. Ignore otherwise. --- ra-dns-check.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 8cacd79..178e1be 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -912,7 +912,9 @@ def load_probe_properties(probe_ids, ppcf): def dict_string(d): dict_str = "" for i,v in d.items(): - dict_str += str(i) + "=\"" + str(v) + "\"," + # Ignore the string output if the value of a key is "None" + if v != "None": + dict_str += str(i) + "=\"" + str(v) + "\"," return dict_str.rstrip(",") # END of all function defs @@ -940,8 +942,10 @@ def dict_string(d): ripe_atlas_latency = { 'measurement_id' : str(dnsprobe['msm_id']), 'probe_id' : str(dnsprobe['prb_id']), 'version' : str(dnsprobe['af']), - 'probe_asn' : str(p_probe_properties[probe_num]['asn_v4']), - 'probe_ip' : str(dnsprobe['from']), + 'probe_asn_v4' : str(p_probe_properties[probe_num]['asn_v4']), + 'probe_address_v4' : str(p_probe_properties[probe_num]['address_v4']), + 'probe_asn_v6' : str(p_probe_properties[probe_num]['asn_v6']), + 'probe_address_v6' : str(p_probe_properties[probe_num]['address_v6']), 'country' : str(p_probe_properties[probe_num]['country_code']), 'loc' : str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1]), 'host' : str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0]), From f901aa2db5b3fd62a13f4955c442b872694a0f91 Mon Sep 17 00:00:00 2001 From: sbng Date: Fri, 8 Dec 2023 16:52:34 -0700 Subject: [PATCH 12/33] issues 5 - Force all queries into UTC time (unix time) only --- ra-dns-check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 178e1be..95e1cea 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -423,8 +423,8 @@ def user_datetime_to_valid_unixtime(user_dt_string): # It's not unix time, so try to convert from some data time formats for f in accepted_datetime_formats: try: - # print (user_dt_string + ' / ' + f) - _unixtime_candidate = int(time.mktime(time.strptime(user_dt_string, f))) + # print (user_dt_string + ' / ' + f) and offset the time zone to UTC + _unixtime_candidate = int(time.mktime(time.strptime(user_dt_string, f))) - time.timezone if is_valid_unixtime(_unixtime_candidate): logger.debug('Accepted %i as valid unixtime.\n' % _unixtime_candidate) return (_unixtime_candidate) From a72c567d496374681c552298dbb7b36d1ad3e424 Mon Sep 17 00:00:00 2001 From: sbng Date: Sat, 9 Dec 2023 12:12:01 +0800 Subject: [PATCH 13/33] Optimize the cache loading process, only new probes will be added to existing cache. Content of new probes include lat/lon position which was missing previously --- ra-dns-check.py | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 95e1cea..ba5712a 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -879,32 +879,40 @@ def load_probe_properties(probe_ids, ppcf): # and discover new element not found in the caches. This reduce computational complexity dns_probes = set(str(x) for x in probe_ids) all_probes = set(all_probes_dict.keys()) + #******************* + # remove this after complete testing + #******************* new_probes = dns_probes.difference(all_probes) for i in dns_probes: matched_probe_info[i] = all_probes_dict[i] - try: - for p in new_probes: + for p in new_probes: + try: ripe_result = Probe(id=p) matched_probe_info[p] = {'asn_v4': ripe_result.asn_v4, - 'asn_v6': ripe_result.asn_v6, - 'country_code': ripe_result.country_code, - 'address_v4': ripe_result.address_v4, - 'address_v6': ripe_result.address_v6} + 'asn_v6': ripe_result.asn_v6, + 'country_code': ripe_result.country_code, + 'lat': ripe_result.geometry['coordinates'][1], + 'lon': ripe_result.geometry['coordinates'][0], + 'address_v4': ripe_result.address_v4, + 'address_v6': ripe_result.address_v6} all_probes_dict[p] = matched_probe_info[p] logger.debug('Probe %9s info fetched from RIPE' % p) - except: - # Otherwise, it's empty - # we did not find any information about the probe, so set values to '-' - matched_probe_info[p] = { 'asn_v4': '-', - 'asn_v6': '-', - 'country_code': '-', - 'address_v4': '-', - 'address_v6': '-' } - logger.debug('Failed to get info about probe ID %s in the local cache or from RIPE Atlas API.' % p) + except: + # Otherwise, it's empty + # we did not find any information about the probe, so set values to '-' + matched_probe_info[p] = { 'asn_v4': '-', + 'asn_v6': '-', + 'country_code': '-', + 'lat': '-', + 'lon': '-', + 'address_v4': '-', + 'address_v6': '-' } + logger.debug('Failed to get info about probe ID %s in the local cache or from RIPE Atlas API.' % p) logger.info('cache hits: %i cache misses: %i.\n' % (probe_cache_hits, probe_cache_misses)) # Write out the local JSON cache file - with open(ppcf, mode='w') as f: - json.dump(all_probes_dict, f) + if len(new_probes) != 0: + with open(ppcf, mode='w') as f: + json.dump(all_probes_dict, f) return(matched_probe_info) #################### # From 0c4f50cb96c5ada382125ed7e26e0cee8bd85906 Mon Sep 17 00:00:00 2001 From: sbng Date: Sat, 9 Dec 2023 12:15:13 +0800 Subject: [PATCH 14/33] Cleaned comments in the codes for clarity --- ra-dns-check.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index ba5712a..63d6fa4 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -879,9 +879,6 @@ def load_probe_properties(probe_ids, ppcf): # and discover new element not found in the caches. This reduce computational complexity dns_probes = set(str(x) for x in probe_ids) all_probes = set(all_probes_dict.keys()) - #******************* - # remove this after complete testing - #******************* new_probes = dns_probes.difference(all_probes) for i in dns_probes: matched_probe_info[i] = all_probes_dict[i] From 5b07dffdf4105d081bbfab24ba9a44e9f123ab4f Mon Sep 17 00:00:00 2001 From: sbng Date: Sat, 9 Dec 2023 22:10:22 +0800 Subject: [PATCH 15/33] Update README.md for --scrape output when *NO* datetime1 arguments is provided --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 4805bcb..f0796cc 100644 --- a/README.md +++ b/README.md @@ -47,5 +47,8 @@ don't need to do these again, unless the dependencies have changed. 'rt' of DNS query. The output are formatted to allow Prometheus to scrape. * Additonally, another arugment --probes '' only output the specific probes values offered by the +* if --datetime1 is missing, the latest set of data base on time now() will be output + * ```ra-dns-check.py --datetime1 202210080000 12016241 --scrape``` * ```ra-dns-check.py --datetime1 202210080000 12016241 --scrape --probes "999,99"``` +* ```ra-dns-check.py 64573001 --scrape``` From b3e7e5fbcd9fa5f3accf8faa6e3e6002d0e86480 Mon Sep 17 00:00:00 2001 From: sbng Date: Mon, 11 Dec 2023 06:25:01 +0800 Subject: [PATCH 16/33] Fix issue 7 - Rname labels loc->sample_reported_pop, host->sample_reported_host, country->probe_country, lat->probe_lat, lon->probe_lon --- ra-dns-check.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 63d6fa4..2002bce 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -951,11 +951,11 @@ def dict_string(d): 'probe_address_v4' : str(p_probe_properties[probe_num]['address_v4']), 'probe_asn_v6' : str(p_probe_properties[probe_num]['asn_v6']), 'probe_address_v6' : str(p_probe_properties[probe_num]['address_v6']), - 'country' : str(p_probe_properties[probe_num]['country_code']), - 'loc' : str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1]), - 'host' : str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0]), - 'lat' : str(p_probe_properties[probe_num]['latitude']), - 'lon' : str(p_probe_properties[probe_num]['longitude']), + 'probe_country' : str(p_probe_properties[probe_num]['country_code']), + 'sample_reported_pop' : str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1]), + 'sample_reported_host' : str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0]), + 'probe_lat' : str(p_probe_properties[probe_num]['latitude']), + 'probe_lon' : str(p_probe_properties[probe_num]['longitude']), } labels = dict_string(ripe_atlas_latency) print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') From 043365dedfb1cec88fdf1aa883d66f5343079928 Mon Sep 17 00:00:00 2001 From: sbng Date: Mon, 11 Dec 2023 06:52:33 +0800 Subject: [PATCH 17/33] Issue 6 - add a new argument to allow the printing of timestamp in Prometheus scrape output (default is not to display) --- ra-dns-check.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 2002bce..04285ac 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -189,9 +189,13 @@ 'help': 'The max age (seconds) of the RIPE Atlas probe info file (older than this and we download a new one). Default: 86400', 'type': 'integer'}, 'scrape': { - 'default': '', + 'default': False, 'help': 'Scrape output for Prometheus', - 'type': 'string'}, + 'type': 'boolean'}, + 'include_probe_timestamp': { + 'default': False, + 'help': 'Default timestamp are not display in the Prometheus output, toggle this switch to display timestamp', + 'type': 'boolean'}, 'probes': { 'default': [], 'help': 'Selected probes list eg. "919,166,1049"', @@ -269,6 +273,7 @@ parser.add_argument('-t', '--split_char', help=options_sample_dict['split_char']['help'], type=str, default=options_sample_dict['split_char']['default']) parser.add_argument('-u', '--print_summary_stats', help=options_sample_dict['print_summary_stats']['help'], action='store_true', default=options_sample_dict['print_summary_stats']['default']) parser.add_argument('--scrape', help=options_sample_dict['scrape']['help'], action='store_true', default=options_sample_dict['scrape']['default']) +parser.add_argument('--include_probe_timestamp', help=options_sample_dict['include_probe_timestamp']['help'], action='store_true', default=options_sample_dict['include_probe_timestamp']['default']) parser.add_argument('--probes', help=options_sample_dict['probes']['help'], type=str, default=options_sample_dict['probes']['default']) parser.add_argument('filename_or_msmid', help='one or two local filenames or RIPE Atlas Measurement IDs', nargs='+') parser.format_help() @@ -958,7 +963,10 @@ def dict_string(d): 'probe_lon' : str(p_probe_properties[probe_num]['longitude']), } labels = dict_string(ripe_atlas_latency) - print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') + if args[0].include_probe_timestamp: + print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') + else: + print (f'ripe_atlas_latency{{{labels}}} {delay}') except: pass exit() From e09b1685de1511f8eb715d25d16c4c6b63844908 Mon Sep 17 00:00:00 2001 From: sbng Date: Mon, 11 Dec 2023 07:14:27 +0800 Subject: [PATCH 18/33] Change of --scrape display to include timestamp if --datetime1 are not present Revert "Change of --scrape display to include timestamp if --datetime1 are not present" This reverts commit a6643908ce5110ed39abb7362e75ffeaf0b42821. Change of --scrape display to exclude timestamp if --datetime1 are not present. the --include_probe_timestamp can be toggled if timestamp is required --- ra-dns-check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 04285ac..5c07583 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -963,7 +963,7 @@ def dict_string(d): 'probe_lon' : str(p_probe_properties[probe_num]['longitude']), } labels = dict_string(ripe_atlas_latency) - if args[0].include_probe_timestamp: + if (args[0].include_probe_timestamp) and (args[0].datetime1 != None) : print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') else: print (f'ripe_atlas_latency{{{labels}}} {delay}') From 40b895ef7a984a79978dbc4e9b48874323549c29 Mon Sep 17 00:00:00 2001 From: sbng Date: Fri, 15 Dec 2023 15:26:58 +0800 Subject: [PATCH 19/33] Issue 8. Force the regeneration of the cache file in load_probe_properties() function --- ra-dns-check.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 5c07583..665f156 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -870,12 +870,20 @@ def load_probe_properties(probe_ids, ppcf): all_probes_dict = {} # logger.info ('Reading the probe data dictionary as a JSON file from %s...\n' % ppcf) - try: - with open(ppcf, 'r') as f: - all_probes_dict = json.load(f) - except: - logger.critical ('Cannot read probe data from file: %s\n' % ppcf) - exit(13) + while True: + try: + with open(ppcf, 'r') as f: + all_probes_dict = json.load(f) + except: + logger.critical ('Cannot read probe data from file: %s\n' % ppcf) + logger.critical ('Regenerating probe data to file: %s\n' % ppcf) + _res = check_update_probe_properties_cache_file(config['ripe_atlas_probe_properties_raw_file'], + config['ripe_atlas_probe_properties_json_cache_file'], + config['ripe_atlas_current_probe_properties_url']) + if _res != 0: + logger.critical('Unexpected result when updating local cache files: %s' % _res) + continue + break # Loop through the list of supplied (seen) probe ids and collect their # info/meta data from either our local file or the RIPE Atlas API logger.info ('Matching seen probes with probe data; will query RIPE Atlas API for probe info not in local cache...\n') From cf550de832a5f2f74a89e47b5ac19f1580fe0656 Mon Sep 17 00:00:00 2001 From: sbng Date: Sat, 16 Dec 2023 06:23:29 +0800 Subject: [PATCH 20/33] Print timestamp for --scrape if --include_probe_timestamp is present OR --datetime1 is missing (i.e now() does not need timestamp) --- ra-dns-check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 665f156..db97ea0 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -971,7 +971,7 @@ def dict_string(d): 'probe_lon' : str(p_probe_properties[probe_num]['longitude']), } labels = dict_string(ripe_atlas_latency) - if (args[0].include_probe_timestamp) and (args[0].datetime1 != None) : + if (args[0].include_probe_timestamp) or (args[0].datetime1 != None) : print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') else: print (f'ripe_atlas_latency{{{labels}}} {delay}') From b120a711dae48e241ef48bfe34daba53b2ae1431 Mon Sep 17 00:00:00 2001 From: sbng Date: Sat, 16 Dec 2023 23:14:37 +0800 Subject: [PATCH 21/33] Exclude new probes not found in the cache when building the initial match probe info DB --- ra-dns-check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index db97ea0..db79820 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -893,7 +893,7 @@ def load_probe_properties(probe_ids, ppcf): dns_probes = set(str(x) for x in probe_ids) all_probes = set(all_probes_dict.keys()) new_probes = dns_probes.difference(all_probes) - for i in dns_probes: + for i in (dns_probes - new_probes): matched_probe_info[i] = all_probes_dict[i] for p in new_probes: try: From 069c2c6e84ec4daec8016714fa9753acdaeec251 Mon Sep 17 00:00:00 2001 From: sbng Date: Sat, 16 Dec 2023 23:16:39 +0800 Subject: [PATCH 22/33] Add support for google 8.8.8.8 and cloudfare 1.1.1.1 by ignoring the DNS answer field --- ra-dns-check.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index db79820..3990c18 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -965,18 +965,20 @@ def dict_string(d): 'probe_asn_v6' : str(p_probe_properties[probe_num]['asn_v6']), 'probe_address_v6' : str(p_probe_properties[probe_num]['address_v6']), 'probe_country' : str(p_probe_properties[probe_num]['country_code']), - 'sample_reported_pop' : str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1]), - 'sample_reported_host' : str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0]), 'probe_lat' : str(p_probe_properties[probe_num]['latitude']), 'probe_lon' : str(p_probe_properties[probe_num]['longitude']), - } + } + ripe_atlas_latency['sample_reported_pop'] = str(dnsprobe['result']['answers'][0]['RDATA'][0]) + ripe_atlas_latency['sample_reported_pop'] = str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1]) + ripe_atlas_latency['sample_reported_host'] = str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0]) + except: + pass + finally: labels = dict_string(ripe_atlas_latency) if (args[0].include_probe_timestamp) or (args[0].datetime1 != None) : print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') else: print (f'ripe_atlas_latency{{{labels}}} {delay}') - except: - pass exit() while results_set_id <= last_results_set_id: From e8dfceb8be4a75af492f23dac31d84604f828a54 Mon Sep 17 00:00:00 2001 From: sbng Date: Sat, 16 Dec 2023 23:28:56 +0800 Subject: [PATCH 23/33] Issue 10. Added target_ip (eg 8.8.8.8/1.1.1.1/9.9.9.9) as a new label --- ra-dns-check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ra-dns-check.py b/ra-dns-check.py index 3990c18..3112155 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -960,6 +960,7 @@ def dict_string(d): ripe_atlas_latency = { 'measurement_id' : str(dnsprobe['msm_id']), 'probe_id' : str(dnsprobe['prb_id']), 'version' : str(dnsprobe['af']), + 'target_ip' : str(dnsprobe['dst_addr']), 'probe_asn_v4' : str(p_probe_properties[probe_num]['asn_v4']), 'probe_address_v4' : str(p_probe_properties[probe_num]['address_v4']), 'probe_asn_v6' : str(p_probe_properties[probe_num]['asn_v6']), From 596ac24dc8cef696d2c99747ff46580eb26998a6 Mon Sep 17 00:00:00 2001 From: sbng Date: Sun, 17 Dec 2023 08:01:23 +0800 Subject: [PATCH 24/33] Issue 11 - fix consistent labels (key/values) output for Prometheus --- ra-dns-check.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 3112155..849a5f3 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -931,7 +931,9 @@ def dict_string(d): dict_str = "" for i,v in d.items(): # Ignore the string output if the value of a key is "None" - if v != "None": + if v == "None": + dict_str += str(i) + "=\"" + "" + "\"," + else: dict_str += str(i) + "=\"" + str(v) + "\"," return dict_str.rstrip(",") From a0584c3ec8e6e0040e802166d992222b26b6bd1e Mon Sep 17 00:00:00 2001 From: sbng Date: Tue, 19 Dec 2023 06:16:41 +0800 Subject: [PATCH 25/33] Remove google/cloudflare scrape support as it is causing some logic error --- ra-dns-check.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 849a5f3..7ba13be 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -964,24 +964,22 @@ def dict_string(d): 'version' : str(dnsprobe['af']), 'target_ip' : str(dnsprobe['dst_addr']), 'probe_asn_v4' : str(p_probe_properties[probe_num]['asn_v4']), - 'probe_address_v4' : str(p_probe_properties[probe_num]['address_v4']), + 'probe_address_v4' : str(dnsprobe['from']), 'probe_asn_v6' : str(p_probe_properties[probe_num]['asn_v6']), 'probe_address_v6' : str(p_probe_properties[probe_num]['address_v6']), 'probe_country' : str(p_probe_properties[probe_num]['country_code']), 'probe_lat' : str(p_probe_properties[probe_num]['latitude']), 'probe_lon' : str(p_probe_properties[probe_num]['longitude']), } - ripe_atlas_latency['sample_reported_pop'] = str(dnsprobe['result']['answers'][0]['RDATA'][0]) ripe_atlas_latency['sample_reported_pop'] = str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1]) ripe_atlas_latency['sample_reported_host'] = str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0]) - except: - pass - finally: labels = dict_string(ripe_atlas_latency) if (args[0].include_probe_timestamp) or (args[0].datetime1 != None) : print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') else: print (f'ripe_atlas_latency{{{labels}}} {delay}') + except: + pass exit() while results_set_id <= last_results_set_id: From 5bc2747f051b4addbe229f875244892c9b9eeef9 Mon Sep 17 00:00:00 2001 From: sbng Date: Tue, 19 Dec 2023 21:02:43 +0800 Subject: [PATCH 26/33] add autocomplete support options using argcomplete module, use --autocomplete to egnerate the autocomplete command required for bash/zsh --- ra-dns-check.py | 24 ++++++++++++++++++++++-- requirements.txt | 2 +- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 7ba13be..0dd8b7b 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -6,7 +6,7 @@ # Please see the file LICENSE for the license. -import argparse +import argparse,argcomplete # need ast to more safely parse config file import ast import configparser @@ -31,11 +31,13 @@ from ripe.atlas.cousteau import Measurement # for debugging from pprint import pprint - # # Valid log levels valid_log_levels = ['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'] +# Valid id server method +valid_id_server_method = ['quad9','google','cloudflare'] + # Change "CRITICAL" to DEBUG if you want debugging-level logging *before* this # script parses command line args, and subsequently sets the debug level: logging.basicConfig(level=logging.CRITICAL) @@ -196,10 +198,18 @@ 'default': False, 'help': 'Default timestamp are not display in the Prometheus output, toggle this switch to display timestamp', 'type': 'boolean'}, + 'autocomplete': { + 'default': False, + 'help': 'Autocomplete for all options available', + 'type': 'boolean'}, 'probes': { 'default': [], 'help': 'Selected probes list eg. "919,166,1049"', 'type': 'list'}, + 'id_servermethod': { + 'default': 'quad9', + 'help': 'Select the method to handle id.server, option include [quad9,google,cloudflare]', + 'type': 'string'}, 'exclusion_list_file': { 'default': None, 'help': 'Filename for probe ID exclusion list', @@ -274,9 +284,12 @@ parser.add_argument('-u', '--print_summary_stats', help=options_sample_dict['print_summary_stats']['help'], action='store_true', default=options_sample_dict['print_summary_stats']['default']) parser.add_argument('--scrape', help=options_sample_dict['scrape']['help'], action='store_true', default=options_sample_dict['scrape']['default']) parser.add_argument('--include_probe_timestamp', help=options_sample_dict['include_probe_timestamp']['help'], action='store_true', default=options_sample_dict['include_probe_timestamp']['default']) +parser.add_argument('--autocomplete', help=options_sample_dict['autocomplete']['help'], action='store_true', default=options_sample_dict['autocomplete']['default']) parser.add_argument('--probes', help=options_sample_dict['probes']['help'], type=str, default=options_sample_dict['probes']['default']) +parser.add_argument('--id_servermethod', help=options_sample_dict['id_servermethod']['help'], type=str, choices=valid_id_server_method, default=options_sample_dict['id_servermethod']['default']) parser.add_argument('filename_or_msmid', help='one or two local filenames or RIPE Atlas Measurement IDs', nargs='+') parser.format_help() +argcomplete.autocomplete(parser) args = parser.parse_known_args() ###pprint(args) @@ -521,6 +534,13 @@ class fmt: # def id(self): +# if autocomplete option is set, register all the options for autocomplete then exit +if args[0].autocomplete: + prog = sys.argv[0].removeprefix("./") + print(f'source <(register-python-argcomplete {prog})') + os.system('source <(register-python-argcomplete {prog})') + exit() + # Validate the supplied date-times and stick them in a list if args[0].datetime1: logger.debug(args[0].datetime1) diff --git a/requirements.txt b/requirements.txt index 48c2b27..133f880 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ pycparser==2.20 pyOpenSSL==20.0.1 python-dateutil==2.8.1 pytz==2021.1 -PyYAML==5.4.1 +PyYAML==5.3.1 requests==2.31.0 ripe.atlas.cousteau==1.4.2 ripe.atlas.sagan==1.3.0 From f89048121eaad2d8e5c49592e3b16909bff346c0 Mon Sep 17 00:00:00 2001 From: sbng Date: Tue, 19 Dec 2023 21:03:56 +0800 Subject: [PATCH 27/33] add google and cloudflare support using base64 NSID abuf information to decode pop/host code --- ra-dns-check.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 0dd8b7b..8805a63 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -21,6 +21,7 @@ from datetime import datetime # to decompress RIPE Atlas probe data file import bz2 +import base64 # needed to fetch the probe properties file from RIPE import urllib.request # These RIPE python modules are usually installed with pip: @@ -957,6 +958,20 @@ def dict_string(d): dict_str += str(i) + "=\"" + str(v) + "\"," return dict_str.rstrip(",") +##################### +# +# Input a base64 encoded bytes string -> output the last word of the string +def decode_base64(abuf): + locate_nsid = re.sub(r'(\\x..)|\'|(\\t)',' ',str(abuf)).split() + return locate_nsid[-1] + +##################### +# +# Input any string -> output with santinization of special characters with exception of '-' +def sanitize_string(s): + # Regex to match special characters and remove it however need to preserve '-' + return re.sub(r'\W+','',s.replace('-','_')) + # END of all function defs ################################################## @@ -991,8 +1006,21 @@ def dict_string(d): 'probe_lat' : str(p_probe_properties[probe_num]['latitude']), 'probe_lon' : str(p_probe_properties[probe_num]['longitude']), } - ripe_atlas_latency['sample_reported_pop'] = str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[1]) - ripe_atlas_latency['sample_reported_host'] = str(dnsprobe['result']['answers'][0]['RDATA'][0].split('.')[0]) + match args[0].id_servermethod: + case 'quad9': + nsid = decode_base64(base64.b64decode(str(dnsprobe['result']['abuf']))) + ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(nsid.split('.')[1])) + ripe_atlas_latency['sample_reported_host'] = sanitize_string(str(nsid.split('.')[0])) + case 'cloudflare': + ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(dnsprobe['result']['answers'][0]['RDATA'][0])) + ripe_atlas_latency['sample_reported_host'] = "" + case 'google': + nsid = decode_base64(base64.b64decode(str(dnsprobe['result']['abuf']))) + ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(nsid)) + ripe_atlas_latency['sample_reported_host'] = "" + case _: + ripe_atlas_latency['sample_reported_pop'] = "" + ripe_atlas_latency['sample_reported_host'] = "" labels = dict_string(ripe_atlas_latency) if (args[0].include_probe_timestamp) or (args[0].datetime1 != None) : print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') From 6912ea6bc018664c04bb7b74a0f1c4230fcf2cff Mon Sep 17 00:00:00 2001 From: sbng Date: Tue, 19 Dec 2023 21:26:03 +0800 Subject: [PATCH 28/33] update requirements.txt and README.md to include argcomplete module for autocomplete update README.md on autocomplete feature update README.md on autocomplete feature --- README.md | 6 ++++++ requirements.txt | 1 + 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index f0796cc..b847375 100644 --- a/README.md +++ b/README.md @@ -52,3 +52,9 @@ don't need to do these again, unless the dependencies have changed. * ```ra-dns-check.py --datetime1 202210080000 12016241 --scrape``` * ```ra-dns-check.py --datetime1 202210080000 12016241 --scrape --probes "999,99"``` * ```ra-dns-check.py 64573001 --scrape``` + +* Add autocomplete support via argcomplete module. This feature allows the usage of tab key for command/arguments completion. + +* ```pip install -r requirements.txt``` +* ```./ra-dns-check.py --scrape 43869257 --autocomplete ``` +* ```source <(register-python-argcomplete ra-dns-check.py)``` diff --git a/requirements.txt b/requirements.txt index 133f880..6fa3f35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ socketIO-client==0.7.2 tzlocal==2.1 urllib3==1.26.18 websocket-client==0.58.0 +argcomplete==2.0.0 From 335a0d35ed59726f421e9dde280a517f774b2f71 Mon Sep 17 00:00:00 2001 From: sbng Date: Wed, 20 Dec 2023 05:17:16 +0800 Subject: [PATCH 29/33] Populate 'unknown' as value for ''(null) value in the --scrape output --- ra-dns-check.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 8805a63..4ef7524 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -953,7 +953,7 @@ def dict_string(d): for i,v in d.items(): # Ignore the string output if the value of a key is "None" if v == "None": - dict_str += str(i) + "=\"" + "" + "\"," + dict_str += str(i) + "=\"" + "unknown" + "\"," else: dict_str += str(i) + "=\"" + str(v) + "\"," return dict_str.rstrip(",") @@ -1013,14 +1013,14 @@ def sanitize_string(s): ripe_atlas_latency['sample_reported_host'] = sanitize_string(str(nsid.split('.')[0])) case 'cloudflare': ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(dnsprobe['result']['answers'][0]['RDATA'][0])) - ripe_atlas_latency['sample_reported_host'] = "" + ripe_atlas_latency['sample_reported_host'] = "unknown" case 'google': nsid = decode_base64(base64.b64decode(str(dnsprobe['result']['abuf']))) ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(nsid)) - ripe_atlas_latency['sample_reported_host'] = "" + ripe_atlas_latency['sample_reported_host'] = "unknown" case _: - ripe_atlas_latency['sample_reported_pop'] = "" - ripe_atlas_latency['sample_reported_host'] = "" + ripe_atlas_latency['sample_reported_pop'] = "unknown" + ripe_atlas_latency['sample_reported_host'] = "unknown" labels = dict_string(ripe_atlas_latency) if (args[0].include_probe_timestamp) or (args[0].datetime1 != None) : print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') From 9e4ea856bbd00ea89241a98a41bb3764f991cdeb Mon Sep 17 00:00:00 2001 From: sbng Date: Thu, 21 Dec 2023 13:37:57 +0800 Subject: [PATCH 30/33] Convert python case statement to if/else for users with python version < 3.10 --- ra-dns-check.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 4ef7524..8383cc4 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -1006,19 +1006,19 @@ def sanitize_string(s): 'probe_lat' : str(p_probe_properties[probe_num]['latitude']), 'probe_lon' : str(p_probe_properties[probe_num]['longitude']), } - match args[0].id_servermethod: - case 'quad9': - nsid = decode_base64(base64.b64decode(str(dnsprobe['result']['abuf']))) - ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(nsid.split('.')[1])) - ripe_atlas_latency['sample_reported_host'] = sanitize_string(str(nsid.split('.')[0])) - case 'cloudflare': + method = args[0].id_servermethod + if method == 'quad9': + nsid = decode_base64(base64.b64decode(str(dnsprobe['result']['abuf']))) + ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(nsid.split('.')[1])) + ripe_atlas_latency['sample_reported_host'] = sanitize_string(str(nsid.split('.')[0])) + elif method == 'cloudflare': ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(dnsprobe['result']['answers'][0]['RDATA'][0])) ripe_atlas_latency['sample_reported_host'] = "unknown" - case 'google': + elif method == 'google': nsid = decode_base64(base64.b64decode(str(dnsprobe['result']['abuf']))) ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(nsid)) ripe_atlas_latency['sample_reported_host'] = "unknown" - case _: + else: ripe_atlas_latency['sample_reported_pop'] = "unknown" ripe_atlas_latency['sample_reported_host'] = "unknown" labels = dict_string(ripe_atlas_latency) From 1d13b882bbf3209fab5e81ce343c40612cd9ff09 Mon Sep 17 00:00:00 2001 From: sbng Date: Sat, 23 Dec 2023 20:06:02 +0800 Subject: [PATCH 31/33] Add an extra return value for process_request function (return measurement id and dnsresults) --- ra-dns-check.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 8383cc4..5cbe753 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -1034,7 +1034,7 @@ def sanitize_string(s): #for t in data_sources: # m will receive the measurement ID for the processed data source logger.debug('data_source: %s results_set_id: %i unixtime: %i\n' % (data_sources[results_set_id], results_set_id, unixtimes[results_set_id])) - m = process_request(data_sources[results_set_id], results_set_id, unixtimes[results_set_id], probes) + m, r = process_request(data_sources[results_set_id], results_set_id, unixtimes[results_set_id], probes) measurement_ids.append(m) ###### # Summary stats @@ -1068,7 +1068,6 @@ def sanitize_string(s): # results_set_id += 1 # end of Data loading and summary stats reporting loop - ######################################## # Check to see if there are two sets of results. If there are, see if From 658545bd4e6a75198118f0780b3762fc0f837207 Mon Sep 17 00:00:00 2001 From: sbng Date: Sat, 23 Dec 2023 23:21:16 +0800 Subject: [PATCH 32/33] Feature --scrape-staleness-seconds - provide a staleness time to suppress scrape output that is longer then this time --- ra-dns-check.py | 84 ++++++++++++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/ra-dns-check.py b/ra-dns-check.py index 5cbe753..34aa1f0 100755 --- a/ra-dns-check.py +++ b/ra-dns-check.py @@ -215,6 +215,11 @@ 'default': None, 'help': 'Filename for probe ID exclusion list', 'type': 'string'}, + 'scrape_staleness_seconds': { + 'default': int(time.time()), + 'help': 'Staleness of the data/records that exceed this value would be ignored. Staleness is determine between now() and record timestamp', + 'type': 'integer'}, + } sample_config_string = sample_config_string_header @@ -288,6 +293,7 @@ parser.add_argument('--autocomplete', help=options_sample_dict['autocomplete']['help'], action='store_true', default=options_sample_dict['autocomplete']['default']) parser.add_argument('--probes', help=options_sample_dict['probes']['help'], type=str, default=options_sample_dict['probes']['default']) parser.add_argument('--id_servermethod', help=options_sample_dict['id_servermethod']['help'], type=str, choices=valid_id_server_method, default=options_sample_dict['id_servermethod']['default']) +parser.add_argument('--scrape_staleness_seconds', help=options_sample_dict['scrape_staleness_seconds']['help'], type=int, default=options_sample_dict['scrape_staleness_seconds']['default']) parser.add_argument('filename_or_msmid', help='one or two local filenames or RIPE Atlas Measurement IDs', nargs='+') parser.format_help() argcomplete.autocomplete(parser) @@ -972,6 +978,16 @@ def sanitize_string(s): # Regex to match special characters and remove it however need to preserve '-' return re.sub(r'\W+','',s.replace('-','_')) +##################### +# +# Input any timestamp and staleness -> output None value if timestamp exceed staleness else output timestamp +def check_freshness(timestamp,staleness): + freshness = int(time.time()) - timestamp + if (freshness > staleness): + return None + else: + return timestamp + # END of all function defs ################################################## @@ -985,6 +1001,7 @@ def sanitize_string(s): probes = [] if args[0].scrape: + staleness = args[0].scrape_staleness_seconds m, dnsresult = process_request(data_sources[results_set_id], results_set_id, unixtimes[results_set_id], probes) p_probe_properties = load_probe_properties([dnsresult[x]['prb_id'] for x in range(len(dnsresult))], config['ripe_atlas_probe_properties_json_cache_file']) print ('''# HELP ripe_atlas_latency The number of milliseconds for response reported by this probe for the time period requested on this measurement @@ -993,39 +1010,42 @@ def sanitize_string(s): try: probe_num = str(dnsprobe['prb_id']) delay = dnsprobe['result']['rt'] - timestamp = dnsprobe['timestamp'] - ripe_atlas_latency = { 'measurement_id' : str(dnsprobe['msm_id']), - 'probe_id' : str(dnsprobe['prb_id']), - 'version' : str(dnsprobe['af']), - 'target_ip' : str(dnsprobe['dst_addr']), - 'probe_asn_v4' : str(p_probe_properties[probe_num]['asn_v4']), - 'probe_address_v4' : str(dnsprobe['from']), - 'probe_asn_v6' : str(p_probe_properties[probe_num]['asn_v6']), - 'probe_address_v6' : str(p_probe_properties[probe_num]['address_v6']), - 'probe_country' : str(p_probe_properties[probe_num]['country_code']), - 'probe_lat' : str(p_probe_properties[probe_num]['latitude']), - 'probe_lon' : str(p_probe_properties[probe_num]['longitude']), - } - method = args[0].id_servermethod - if method == 'quad9': - nsid = decode_base64(base64.b64decode(str(dnsprobe['result']['abuf']))) - ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(nsid.split('.')[1])) - ripe_atlas_latency['sample_reported_host'] = sanitize_string(str(nsid.split('.')[0])) - elif method == 'cloudflare': - ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(dnsprobe['result']['answers'][0]['RDATA'][0])) - ripe_atlas_latency['sample_reported_host'] = "unknown" - elif method == 'google': - nsid = decode_base64(base64.b64decode(str(dnsprobe['result']['abuf']))) - ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(nsid)) - ripe_atlas_latency['sample_reported_host'] = "unknown" - else: - ripe_atlas_latency['sample_reported_pop'] = "unknown" - ripe_atlas_latency['sample_reported_host'] = "unknown" - labels = dict_string(ripe_atlas_latency) - if (args[0].include_probe_timestamp) or (args[0].datetime1 != None) : - print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') + timestamp = check_freshness(dnsprobe['timestamp'],staleness) + if timestamp != None: + ripe_atlas_latency = { 'measurement_id' : str(dnsprobe['msm_id']), + 'probe_id' : str(dnsprobe['prb_id']), + 'version' : str(dnsprobe['af']), + 'target_ip' : str(dnsprobe['dst_addr']), + 'probe_asn_v4' : str(p_probe_properties[probe_num]['asn_v4']), + 'probe_address_v4' : str(dnsprobe['from']), + 'probe_asn_v6' : str(p_probe_properties[probe_num]['asn_v6']), + 'probe_address_v6' : str(p_probe_properties[probe_num]['address_v6']), + 'probe_country' : str(p_probe_properties[probe_num]['country_code']), + 'probe_lat' : str(p_probe_properties[probe_num]['latitude']), + 'probe_lon' : str(p_probe_properties[probe_num]['longitude']), + } + method = args[0].id_servermethod + if method == 'quad9': + nsid = decode_base64(base64.b64decode(str(dnsprobe['result']['abuf']))) + ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(nsid.split('.')[1])) + ripe_atlas_latency['sample_reported_host'] = sanitize_string(str(nsid.split('.')[0])) + elif method == 'cloudflare': + ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(dnsprobe['result']['answers'][0]['RDATA'][0])) + ripe_atlas_latency['sample_reported_host'] = "unknown" + elif method == 'google': + nsid = decode_base64(base64.b64decode(str(dnsprobe['result']['abuf']))) + ripe_atlas_latency['sample_reported_pop'] = sanitize_string(str(nsid)) + ripe_atlas_latency['sample_reported_host'] = "unknown" + else: + ripe_atlas_latency['sample_reported_pop'] = "unknown" + ripe_atlas_latency['sample_reported_host'] = "unknown" + labels = dict_string(ripe_atlas_latency) + if (args[0].include_probe_timestamp) or (args[0].datetime1 != None) : + print (f'ripe_atlas_latency{{{labels}}} {delay} {timestamp}') + else: + print (f'ripe_atlas_latency{{{labels}}} {delay}') else: - print (f'ripe_atlas_latency{{{labels}}} {delay}') + logger.info('Skipping probe %s - sample is %i seconds old' % (probe_num,int(time.time() - dnsprobe['timestamp']))) except: pass exit() From 00c1ed651b14b0f90fec89315ea69288f403ea51 Mon Sep 17 00:00:00 2001 From: sbng Date: Fri, 29 Dec 2023 08:32:36 +0800 Subject: [PATCH 33/33] Update README.md documenting the new scrape stalness feature --- README.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 51f927d..1f3a05b 100644 --- a/README.md +++ b/README.md @@ -57,18 +57,24 @@ don't need to do these again, unless the dependencies have changed. ## Additional feature -* Added a --scrape argument. This argument will use the time provided (dtime1) variable to obtain the + * Added a --scrape argument. This argument will use the time provided (dtime1) variable to obtain the 'rt' of DNS query. The output are formatted to allow Prometheus to scrape. -* Additonally, another arugment --probes '' only output the specific probes values offered + * Additonally, another arugment --probes '' only output the specific probes values offered by the -* if --datetime1 is missing, the latest set of data base on time now() will be output - -* ```ra-dns-check.py --datetime1 202210080000 12016241 --scrape``` -* ```ra-dns-check.py --datetime1 202210080000 12016241 --scrape --probes "999,99"``` -* ```ra-dns-check.py 64573001 --scrape``` - -* Add autocomplete support via argcomplete module. This feature allows the usage of tab key for command/arguments completion. - -* ```pip install -r requirements.txt``` -* ```./ra-dns-check.py --scrape 43869257 --autocomplete ``` -* ```source <(register-python-argcomplete ra-dns-check.py)``` + * if --datetime1 is missing, the latest set of data base on time now() will be output +``` +ra-dns-check.py --datetime1 202210080000 12016241 --scrape +ra-dns-check.py --datetime1 202210080000 12016241 --scrape --probes "999,99" +ra-dns-check.py 64573001 --scrape +``` + * Add autocomplete support via argcomplete module. This feature allows the usage of tab key for command/arguments completion. +``` +pip install -r requirements.txt +ra-dns-check.py --scrape 43869257 --autocomplete +source <(register-python-argcomplete ra-dns-check.py) +``` +* Enforce staleness of data under the Prometheus scrape feature. Staleness is defined period of the data that would be scraped between now() and the staleness period (in seconds). Stale data will be log as log level info. +``` +# this command pull the latest data on Oct 8 2022 for measurement 38588031 (probe 928 abd 975) +ra-dns-check.py --datetime1 202210080000 12016241 --scrape --scrape_staleness_seconds 38588031 --probes "928,975" --log_level INFO +```