From ea474fd1f47ebfc1a279105e5a98c860f5145d7c Mon Sep 17 00:00:00 2001 From: Digvijaysing Date: Mon, 30 Mar 2026 15:38:14 +0530 Subject: [PATCH 1/3] fix: Extracting timezone from cisco_ios logs --- .../app-almost-syslog-cisco_syslog.conf | 27 ++++++++++++++++--- .../cisco/app-almost-syslog-cisco_syslog.conf | 27 ++++++++++++++++--- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_syslog.conf b/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_syslog.conf index 72da93bb0d..11d41a0715 100644 --- a/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_syslog.conf +++ b/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_syslog.conf @@ -88,10 +88,14 @@ block parser app-almost-syslog-cisco_syslog() { parser { regexp-parser( prefix(".tmp.") - patterns('(?[\*\.])?(?:(?\d+-\d+-\d+T\d+:\d+:\d+(?:\.\d+)?(?:Z|[\+-] *\d+:\d+)|[A-Z][a-z]{2} [ 0123]\d(?: \d{4})? \d\d:\d\d:\d\d(?: [AP]M)?(?:[^ :]+)?)(?: (?[A-Za-z]{1,4}T))?)') + patterns('(?[\*\.])?(?:(?[A-Z][a-z]{2} [ 0123]\d \d{4} \d\d:\d\d:\d\d\.\d{3} [A-Z]{3,4})|(?\d+-\d+-\d+T\d+:\d+:\d+.*))') template('${.tmp.header}') ); }; + rewrite { + set("${.tmp.timestamp}" value(".tmp.date_only")); + subst(" [A-Z]{3,4}$", "", value(".tmp.date_only")); + }; if { filter { "${.tmp.timestatus}" eq "." @@ -106,16 +110,33 @@ block parser app-almost-syslog-cisco_syslog() { }; }; parser { date-parser-nofilter(format( + '%b %d %Y %H:%M:%S.%f %z', + '%b %e %Y %H:%M:%S.%f %z', '%b %d %H:%M:%S.%f', '%b %d %H:%M:%S', '%b %d %I:%M:%S %p.%f', '%b %d %I:%M:%S %p', - '%b %d %Y %I:%M:%S %p.%f' + '%b %d %Y %I:%M:%S %p.%f', '%b %d %Y %H:%M:%S.%f', '%b %d %Y %H:%M:%S', '%Y-%m-%dT%T%z', ) - template("${.tmp.timestamp}")); + template("${.tmp.date_only}")); + }; + if (match("CEST|CET" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("Europe/Berlin"); }; + } elif (match("PST|PDT" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("America/Los_Angeles"); }; + } elif (match("EST|EDT" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("America/New_York"); }; + } elif (match("BST|GMT" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("Europe/London"); }; + } elif (match("IST" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("Asia/Kolkata"); }; + } elif (match("JST" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("Asia/Tokyo"); }; + } else { + rewrite { fix-time-zone("UTC"); }; }; } elif { #This is "uptime" if we match this isn't a time stamp diff --git a/package/lite/etc/addons/cisco/app-almost-syslog-cisco_syslog.conf b/package/lite/etc/addons/cisco/app-almost-syslog-cisco_syslog.conf index 72da93bb0d..11d41a0715 100644 --- a/package/lite/etc/addons/cisco/app-almost-syslog-cisco_syslog.conf +++ b/package/lite/etc/addons/cisco/app-almost-syslog-cisco_syslog.conf @@ -88,10 +88,14 @@ block parser app-almost-syslog-cisco_syslog() { parser { regexp-parser( prefix(".tmp.") - patterns('(?[\*\.])?(?:(?\d+-\d+-\d+T\d+:\d+:\d+(?:\.\d+)?(?:Z|[\+-] *\d+:\d+)|[A-Z][a-z]{2} [ 0123]\d(?: \d{4})? \d\d:\d\d:\d\d(?: [AP]M)?(?:[^ :]+)?)(?: (?[A-Za-z]{1,4}T))?)') + patterns('(?[\*\.])?(?:(?[A-Z][a-z]{2} [ 0123]\d \d{4} \d\d:\d\d:\d\d\.\d{3} [A-Z]{3,4})|(?\d+-\d+-\d+T\d+:\d+:\d+.*))') template('${.tmp.header}') ); }; + rewrite { + set("${.tmp.timestamp}" value(".tmp.date_only")); + subst(" [A-Z]{3,4}$", "", value(".tmp.date_only")); + }; if { filter { "${.tmp.timestatus}" eq "." @@ -106,16 +110,33 @@ block parser app-almost-syslog-cisco_syslog() { }; }; parser { date-parser-nofilter(format( + '%b %d %Y %H:%M:%S.%f %z', + '%b %e %Y %H:%M:%S.%f %z', '%b %d %H:%M:%S.%f', '%b %d %H:%M:%S', '%b %d %I:%M:%S %p.%f', '%b %d %I:%M:%S %p', - '%b %d %Y %I:%M:%S %p.%f' + '%b %d %Y %I:%M:%S %p.%f', '%b %d %Y %H:%M:%S.%f', '%b %d %Y %H:%M:%S', '%Y-%m-%dT%T%z', ) - template("${.tmp.timestamp}")); + template("${.tmp.date_only}")); + }; + if (match("CEST|CET" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("Europe/Berlin"); }; + } elif (match("PST|PDT" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("America/Los_Angeles"); }; + } elif (match("EST|EDT" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("America/New_York"); }; + } elif (match("BST|GMT" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("Europe/London"); }; + } elif (match("IST" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("Asia/Kolkata"); }; + } elif (match("JST" value(".tmp.timestamp"))) { + rewrite { fix-time-zone("Asia/Tokyo"); }; + } else { + rewrite { fix-time-zone("UTC"); }; }; } elif { #This is "uptime" if we match this isn't a time stamp From b03bbf27c6999f39f72fe148178f30678d990e04 Mon Sep 17 00:00:00 2001 From: Digvijaysing Date: Thu, 2 Apr 2026 14:00:17 +0530 Subject: [PATCH 2/3] fix: Fixing timezone issue in test cases --- .../app-almost-syslog-cisco_syslog.conf | 28 ++++------ tests/test_cisco_ios.py | 54 ++++++++++++++++--- 2 files changed, 59 insertions(+), 23 deletions(-) diff --git a/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_syslog.conf b/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_syslog.conf index 11d41a0715..d1b76f02ab 100644 --- a/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_syslog.conf +++ b/package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_syslog.conf @@ -88,14 +88,10 @@ block parser app-almost-syslog-cisco_syslog() { parser { regexp-parser( prefix(".tmp.") - patterns('(?[\*\.])?(?:(?[A-Z][a-z]{2} [ 0123]\d \d{4} \d\d:\d\d:\d\d\.\d{3} [A-Z]{3,4})|(?\d+-\d+-\d+T\d+:\d+:\d+.*))') + patterns('(?[\*\.])?(?:(?\d+-\d+-\d+T\d+:\d+:\d+(?:\.\d+)?(?:Z|[\+-] *\d+:\d+)|[A-Z][a-z]{2} [ 0123]\d(?: \d{4})? \d\d:\d\d:\d\d(?: [AP]M)?(?:[^ :]+)?)(?: (?[A-Za-z]{1,4}))?)') template('${.tmp.header}') ); }; - rewrite { - set("${.tmp.timestamp}" value(".tmp.date_only")); - subst(" [A-Z]{3,4}$", "", value(".tmp.date_only")); - }; if { filter { "${.tmp.timestatus}" eq "." @@ -110,8 +106,6 @@ block parser app-almost-syslog-cisco_syslog() { }; }; parser { date-parser-nofilter(format( - '%b %d %Y %H:%M:%S.%f %z', - '%b %e %Y %H:%M:%S.%f %z', '%b %d %H:%M:%S.%f', '%b %d %H:%M:%S', '%b %d %I:%M:%S %p.%f', @@ -121,23 +115,23 @@ block parser app-almost-syslog-cisco_syslog() { '%b %d %Y %H:%M:%S', '%Y-%m-%dT%T%z', ) - template("${.tmp.date_only}")); + template("${.tmp.timestamp}")); }; - if (match("CEST|CET" value(".tmp.timestamp"))) { + if (match("CEST|CET" value(".tmp.tz"))) { rewrite { fix-time-zone("Europe/Berlin"); }; - } elif (match("PST|PDT" value(".tmp.timestamp"))) { + } elif (match("PST|PDT" value(".tmp.tz"))) { rewrite { fix-time-zone("America/Los_Angeles"); }; - } elif (match("EST|EDT" value(".tmp.timestamp"))) { + } elif (match("EST|EDT|DST" value(".tmp.tz"))) { rewrite { fix-time-zone("America/New_York"); }; - } elif (match("BST|GMT" value(".tmp.timestamp"))) { + } elif (match("BST|GMT" value(".tmp.tz"))) { rewrite { fix-time-zone("Europe/London"); }; - } elif (match("IST" value(".tmp.timestamp"))) { + } elif (match("CDT" value(".tmp.tz"))) { + rewrite {fix-time-zone("America/Chicago"); }; + } elif (match("IST" value(".tmp.tz"))) { rewrite { fix-time-zone("Asia/Kolkata"); }; - } elif (match("JST" value(".tmp.timestamp"))) { + } elif (match("JST" value(".tmp.tz"))) { rewrite { fix-time-zone("Asia/Tokyo"); }; - } else { - rewrite { fix-time-zone("UTC"); }; - }; + } else {}; } elif { #This is "uptime" if we match this isn't a time stamp parser { diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index f7f2ceb95c..9c88f00eef 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -9,6 +9,7 @@ from .sendmessage import sendsingle from .splunkutils import splunk_single from .timeutils import time_operations +from zoneinfo import ZoneInfo import datetime import pytest @@ -73,6 +74,43 @@ ] +def return_timezone(message): + tz_list = ['CDT', 'DST', 'EDT', 'CEST', 'CET','PST','PDT','EST','BST','GMT','IST', 'JST'] + return next((tz for tz in tz_list if tz in message), 'UTC') + + +def epoch_to_utc(epoch, abbr): + """ + Converts a epoch timestamp to a UTC datetime object based on a + specific timezone abbreviation and return converted epoch time. + """ + tz_map = { + 'CDT': 'America/Chicago', + 'DST': 'America/New_York', + 'EDT': 'America/New_York', + 'EST': 'America/New_York', + 'CEST': 'Europe/Paris', + 'CET': 'Europe/Paris', + 'PST': 'America/Los_Angeles', + 'PDT': 'America/Los_Angeles', + 'BST': 'Europe/London', + 'GMT': 'Europe/London', + 'IST': 'Asia/Kolkata', + 'JST': 'Asia/Tokyo', + } + + # Get the standard IANA name from your map + iana_name = tz_map.get(abbr) + if not iana_name: + return epoch + + utc_dt = datetime.datetime.fromtimestamp(epoch, datetime.timezone.utc) + local_dt = utc_dt.replace(tzinfo=ZoneInfo(iana_name)) + converted_utc_dt = local_dt.astimezone(datetime.timezone.utc) + + return int(converted_utc_dt.timestamp()) + + @pytest.mark.parametrize("event", testdata) @pytest.mark.addons("cisco") def test_cisco_ios( @@ -102,13 +140,14 @@ def test_cisco_ios( host=host, year=year, ) - + tzname = return_timezone(message) + new_epoch = epoch_to_utc(int(epoch), tzname) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( 'search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' ) - search = st.render(epoch=epoch, millisec=millisec, microsec=microsec, host=host) + search = st.render(epoch=new_epoch, millisec=millisec, microsec=microsec, host=host) result_count, _ = splunk_single(setup_splunk, search) @@ -148,13 +187,14 @@ def test_cisco_ios_badtime( tzname=tzname, host=host, ) - + tzname = return_timezone(message) + new_epoch = epoch_to_utc(int(epoch), tzname) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search index=netops earliest=-1m@m latest=+1m@m sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' + 'search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' ) - search = st.render(host=host) + search = st.render(epoch=new_epoch, millisec=millisec, microsec=microsec, host=host) result_count, _ = splunk_single(setup_splunk, search) @@ -245,13 +285,15 @@ def test_cisco_nx_os_soup2( message = mt.render( mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset ) + tzname = return_timezone(message) + new_epoch = epoch_to_utc(int(epoch), tzname) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( 'search _time={{ epoch }} host!=GMT index=netops sourcetype="cisco:ios" {{ host }}' ) - search = st.render(epoch=epoch, host=host) + search = st.render(epoch=new_epoch, host=host) result_count, _ = splunk_single(setup_splunk, search) From 4dc873fba43f5be85f1f4651fddc0838b276468d Mon Sep 17 00:00:00 2001 From: Digvijaysing Date: Thu, 2 Apr 2026 16:21:34 +0530 Subject: [PATCH 3/3] fix: Changes updated in lite package --- .../cisco/app-almost-syslog-cisco_syslog.conf | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/package/lite/etc/addons/cisco/app-almost-syslog-cisco_syslog.conf b/package/lite/etc/addons/cisco/app-almost-syslog-cisco_syslog.conf index 11d41a0715..d1b76f02ab 100644 --- a/package/lite/etc/addons/cisco/app-almost-syslog-cisco_syslog.conf +++ b/package/lite/etc/addons/cisco/app-almost-syslog-cisco_syslog.conf @@ -88,14 +88,10 @@ block parser app-almost-syslog-cisco_syslog() { parser { regexp-parser( prefix(".tmp.") - patterns('(?[\*\.])?(?:(?[A-Z][a-z]{2} [ 0123]\d \d{4} \d\d:\d\d:\d\d\.\d{3} [A-Z]{3,4})|(?\d+-\d+-\d+T\d+:\d+:\d+.*))') + patterns('(?[\*\.])?(?:(?\d+-\d+-\d+T\d+:\d+:\d+(?:\.\d+)?(?:Z|[\+-] *\d+:\d+)|[A-Z][a-z]{2} [ 0123]\d(?: \d{4})? \d\d:\d\d:\d\d(?: [AP]M)?(?:[^ :]+)?)(?: (?[A-Za-z]{1,4}))?)') template('${.tmp.header}') ); }; - rewrite { - set("${.tmp.timestamp}" value(".tmp.date_only")); - subst(" [A-Z]{3,4}$", "", value(".tmp.date_only")); - }; if { filter { "${.tmp.timestatus}" eq "." @@ -110,8 +106,6 @@ block parser app-almost-syslog-cisco_syslog() { }; }; parser { date-parser-nofilter(format( - '%b %d %Y %H:%M:%S.%f %z', - '%b %e %Y %H:%M:%S.%f %z', '%b %d %H:%M:%S.%f', '%b %d %H:%M:%S', '%b %d %I:%M:%S %p.%f', @@ -121,23 +115,23 @@ block parser app-almost-syslog-cisco_syslog() { '%b %d %Y %H:%M:%S', '%Y-%m-%dT%T%z', ) - template("${.tmp.date_only}")); + template("${.tmp.timestamp}")); }; - if (match("CEST|CET" value(".tmp.timestamp"))) { + if (match("CEST|CET" value(".tmp.tz"))) { rewrite { fix-time-zone("Europe/Berlin"); }; - } elif (match("PST|PDT" value(".tmp.timestamp"))) { + } elif (match("PST|PDT" value(".tmp.tz"))) { rewrite { fix-time-zone("America/Los_Angeles"); }; - } elif (match("EST|EDT" value(".tmp.timestamp"))) { + } elif (match("EST|EDT|DST" value(".tmp.tz"))) { rewrite { fix-time-zone("America/New_York"); }; - } elif (match("BST|GMT" value(".tmp.timestamp"))) { + } elif (match("BST|GMT" value(".tmp.tz"))) { rewrite { fix-time-zone("Europe/London"); }; - } elif (match("IST" value(".tmp.timestamp"))) { + } elif (match("CDT" value(".tmp.tz"))) { + rewrite {fix-time-zone("America/Chicago"); }; + } elif (match("IST" value(".tmp.tz"))) { rewrite { fix-time-zone("Asia/Kolkata"); }; - } elif (match("JST" value(".tmp.timestamp"))) { + } elif (match("JST" value(".tmp.tz"))) { rewrite { fix-time-zone("Asia/Tokyo"); }; - } else { - rewrite { fix-time-zone("UTC"); }; - }; + } else {}; } elif { #This is "uptime" if we match this isn't a time stamp parser {