From 3b4a4c1fcc3d37510bd111324f94bcf702c32dea Mon Sep 17 00:00:00 2001 From: Lorenzo Leonardini Date: Thu, 3 Apr 2025 16:07:00 +0200 Subject: [PATCH 1/7] add pcap subfolders support - add recursive check for existing pcaps at startup - add recursive inode event watch --- services/go-importer/cmd/assembler/main.go | 68 +++++++++++++++++----- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/services/go-importer/cmd/assembler/main.go b/services/go-importer/cmd/assembler/main.go index 8962d8c8..3b53a7d9 100644 --- a/services/go-importer/cmd/assembler/main.go +++ b/services/go-importer/cmd/assembler/main.go @@ -329,29 +329,57 @@ func connectToPCAPOverIP(service AssemblerService, pcapIP string) { } } -func (service AssemblerService) WatchDir(watch_dir string) { - stat, err := os.Stat(watch_dir) +func (service AssemblerService) recursiveHandleExistingFiles(directory string) { + log.Println("Handling already existing files in ", directory) + files, err := ioutil.ReadDir(directory) if err != nil { - log.Fatal("Failed to open the watch_dir with error: ", err) + log.Fatal(err) } - if !stat.IsDir() { - log.Fatal("watch_dir is not a directory") + for _, file := range files { + if file.IsDir() { + service.recursiveHandleExistingFiles(filepath.Join(directory, file.Name())) + continue + } + // accepts files with prefixes that start with .pcap (.pcapng .pcap1 etc) + if strings.HasPrefix(filepath.Ext(file.Name()), ".pcap") { + service.HandlePcapUri(filepath.Join(directory, file.Name())) //FIXME; this is a little clunky + } } +} - log.Println("Monitoring dir: ", watch_dir) +func (service AssemblerService) recursiveWatchDir(watcher *fsnotify.Watcher, directory string) { + log.Println("Watching inode events for ", directory) + err := watcher.Add(directory) + if err != nil { + log.Fatal(err) + } - files, err := ioutil.ReadDir(watch_dir) + files, err := ioutil.ReadDir(directory) if err != nil { log.Fatal(err) } for _, file := range files { - // accepts files with prefixes that start with .pcap (.pcapng .pcap1 etc) - if strings.HasPrefix(filepath.Ext(file.Name()), ".pcap") { - service.HandlePcapUri(filepath.Join(watch_dir, file.Name())) //FIXME; this is a little clunky + if file.IsDir() { + service.recursiveWatchDir(watcher, filepath.Join(directory, file.Name())) } } +} + +func (service AssemblerService) WatchDir(watch_dir string) { + stat, err := os.Stat(watch_dir) + if err != nil { + log.Fatal("Failed to open the watch_dir with error: ", err) + } + + if !stat.IsDir() { + log.Fatal("watch_dir is not a directory") + } + + log.Println("Monitoring dir: ", watch_dir) + + service.recursiveHandleExistingFiles(watch_dir) watcher, err := fsnotify.NewWatcher() if err != nil { @@ -376,6 +404,21 @@ func (service AssemblerService) WatchDir(watch_dir string) { log.Println("Found new file", event.Name, event.Op.String()) time.Sleep(2 * time.Second) // FIXME; bit of race here between file creation and writes. service.HandlePcapUri(event.Name) + } else { + // test if the file is a directory + stat, err := os.Stat(event.Name) + if err != nil { + log.Fatal("Failed to open the watch_dir with error: ", err) + } else if stat.IsDir() { + if event.Op&fsnotify.Rename != 0 { + watcher.Remove(event.Name) + log.Println("Removed watch for ", event.Name) + } else if event.Op&fsnotify.Create != 0 { + watcher.Add(event.Name) + log.Println("Added watch for ", event.Name) + service.recursiveHandleExistingFiles(event.Name) + } + } } } case err, ok := <-watcher.Errors: @@ -387,10 +430,7 @@ func (service AssemblerService) WatchDir(watch_dir string) { } }() - err = watcher.Add(watch_dir) - if err != nil { - log.Fatal(err) - } + service.recursiveWatchDir(watcher, watch_dir) <-signalChan log.Println("Watcher stopped") From 73768f53fabe6722b2bc6fdfc1c51f4df2d6a30c Mon Sep 17 00:00:00 2001 From: Lorenzo Leonardini Date: Thu, 3 Apr 2025 16:08:12 +0200 Subject: [PATCH 2/7] fix broken tulip --- frontend/src/pages/FlowView.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/FlowView.tsx b/frontend/src/pages/FlowView.tsx index 1b5e42df..654cff36 100644 --- a/frontend/src/pages/FlowView.tsx +++ b/frontend/src/pages/FlowView.tsx @@ -351,7 +351,7 @@ function FlowOverview({ flow }: { flow: FullFlow }) {
[{flow.tags.join(", ")}]
Flags:
- [{flow.flags.map((query, i) => ( + [{flow.flags?.map((query, i) => ( {i > 0 ? ', ' : ''} +

After clicking on a flow, press 'w' to scroll to it in flow list

diff --git a/frontend/src/components/FlowList.tsx b/frontend/src/components/FlowList.tsx index b20272a7..0247b13e 100644 --- a/frontend/src/components/FlowList.tsx +++ b/frontend/src/components/FlowList.tsx @@ -13,6 +13,7 @@ import { START_FILTER_KEY, END_FILTER_KEY, FLOW_LIST_REFETCH_INTERVAL_MS, + FORCE_REFETCH_ON_STAR, } from "../const"; import { useAppSelector, useAppDispatch } from "../store"; import { toggleFilterTag } from "../store/filter"; @@ -97,6 +98,7 @@ export function FlowList() { const onHeartHandler = async (flow: Flow) => { await starFlow({ id: flow._id.$oid, star: !flow.tags.includes("starred") }); + if(FORCE_REFETCH_ON_STAR) refetch(); }; const navigate = useNavigate(); @@ -143,7 +145,29 @@ export function FlowList() { [transformedFlowData] ) + useHotkeys('x', async () => { + if(transformedFlowData) { + let flow = transformedFlowData[flowIndex ?? 0] + await onHeartHandler(flow); + } + }) + useHotkeys('j', () => setFlowIndex(fi => Math.min((transformedFlowData?.length ?? 1)-1, fi + 1)), [transformedFlowData?.length]); + useHotkeys('w', () => { + if(transformedFlowData) { + let idAtIndex = transformedFlowData[flowIndex ?? 0]._id.$oid; + if (idAtIndex != openedFlowID) { + let flowids = flowData?.map((flow, idx) => ([flow._id.$oid, idx])) + if (flowids) { + let found = flowids.filter((el)=>(el[0] == openedFlowID)) + if (found.length > 0) { + let n = Number(found[0][1]) + setFlowIndex(n) + } + } + } + } + }); useHotkeys('k', () => setFlowIndex(fi => Math.max(0, fi - 1))); useHotkeys('i', () => { setShowFilters(true) @@ -157,6 +181,12 @@ export function FlowList() { dispatch(toggleFilterTag("flag-out")) } }, [availableTags]); + useHotkeys('t', () => { + setShowFilters(true) + if ((availableTags ?? []).includes("starred")) { + dispatch(toggleFilterTag("starred")) + } + }, [availableTags]); useHotkeys('r', () => refetch()); const [showFilters, setShowFilters] = useState(false); diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx index 8c5b010b..ce0c969b 100644 --- a/frontend/src/components/Header.tsx +++ b/frontend/src/components/Header.tsx @@ -269,7 +269,7 @@ function SecondDiff() { setSearchParams(searchParams); } - useHotkeys("g", () => { + useHotkeys("e", () => { setSecondDiffFlow(); }); @@ -322,6 +322,9 @@ export function Header() { let [searchParams] = useSearchParams(); const { setToLastnTicks, currentTick, setTimeParam } = useMessyTimeStuff(); + let navigate = useNavigate(); + + useHotkeys('g', () => navigate(`/corrie?${searchParams}`, { replace: true })); useHotkeys('a', () => setToLastnTicks(5)); useHotkeys('c', () => { (document.getElementById("startdateselection") as HTMLInputElement).value = ""; diff --git a/frontend/src/const.ts b/frontend/src/const.ts index b2bea039..f6cda876 100644 --- a/frontend/src/const.ts +++ b/frontend/src/const.ts @@ -12,3 +12,5 @@ export const SERVICE_REFETCH_INTERVAL_MS = 15000; export const TICK_REFETCH_INTERVAL_MS = 10000; export const FLOW_LIST_REFETCH_INTERVAL_MS = 30000; export const MAX_LENGTH_FOR_HIGHLIGHT = 400000; + +export const FORCE_REFETCH_ON_STAR = true; \ No newline at end of file diff --git a/frontend/src/pages/FlowView.tsx b/frontend/src/pages/FlowView.tsx index 654cff36..c00d9659 100644 --- a/frontend/src/pages/FlowView.tsx +++ b/frontend/src/pages/FlowView.tsx @@ -458,17 +458,23 @@ export function FlowView() { // TODO: account for user scrolling - update currentFlow accordingly const [currentFlow, setCurrentFlow] = useState(-1); + // reset scroll on flow switch + useHotkeys('j', () => setCurrentFlow(0)) + useHotkeys('k', () => setCurrentFlow(0)) + useHotkeys('h', () => { // we do this for the scroll to top if (currentFlow === 0) { - document.getElementById(`${id}-${currentFlow}`)?.scrollIntoView(true) + // document.getElementById(`${id}-${currentFlow}`)?.scrollIntoView(true) + let el = document.querySelector("main > div > div:nth-child(2)") + if (el) el.scrollIntoView() } setCurrentFlow(fi => Math.max(0, fi - 1)) }, [currentFlow]); useHotkeys('l', () => { - if (currentFlow === (flow?.flow?.length ?? 1)-1) { - document.getElementById(`${id}-${currentFlow}`)?.scrollIntoView(true) - } + // if (currentFlow === (flow?.flow?.length ?? 1)-1) { + // document.getElementById(`${id}-${currentFlow}`)?.scrollIntoView(true) + // } setCurrentFlow(fi => Math.min((flow?.flow?.length ?? 1)-1, fi + 1)) }, [currentFlow, flow?.flow?.length]); @@ -477,7 +483,7 @@ export function FlowView() { if (currentFlow < 0) { return } - document.getElementById(`${id}`)?.scrollIntoView(true) + document.getElementById(`${id}-${currentFlow}`)?.scrollIntoView(true) }, [currentFlow] ) diff --git a/frontend/src/pages/Home.tsx b/frontend/src/pages/Home.tsx index 8d1438df..7b38437f 100644 --- a/frontend/src/pages/Home.tsx +++ b/frontend/src/pages/Home.tsx @@ -1,20 +1,26 @@ const shortcutTableData = [ [ { key: 'j/k', action: 'Down/Up in FlowList' }, - { key: 's', action: 'Focus search bar' }, + { key: 'h/l', action: 'Up/Down in Flow' }, + { key: 's', action: 'Focus (s)earch bar' }, { key: 'esc', action: 'Unfocus search bar' }, - { key: 'i/o', action: 'Toggle flag in/out filters' }, ], [ - { key: 'h/l', action: 'Up/Down in Flow' }, - { key: 'a', action: 'Last 5 ticks' }, - { key: 'c', action: 'Clear time selection' }, - { key: 'r', action: 'Refresh flows' }, + { key: 'a', action: 'L(a)st 5 ticks' }, + { key: 'c', action: '(C)lear time selection' }, + { key: 'r', action: '(R)efresh flows' }, ], [ - { key: 'd', action: 'Diff view' }, - { key: 'f', action: 'Load flow to first diff slot' }, - { key: 'g', action: 'Load flow to second diff slot' }, + { key: 'd', action: '(D)iff view' }, + { key: 'f', action: 'Load flow to (f)irst diff slot' }, + { key: 'e', action: 'Load flow to s(e)ond diff slot' }, + { key: 'g', action: '(G)raph view' }, + ], + [ + { key: 'w', action: 'Scroll to current flo(w) in flow list' }, + { key: 'i/o', action: 'Toggle flag in/out filters' }, + { key: 't', action: 'Toggle s(t)arred filters' }, + { key: 'x', action: 'Star selected flow' }, ] ]; From 6b605ae984c3a98d026a5405ad06d02901d6c905 Mon Sep 17 00:00:00 2001 From: Lorenzo Leonardini Date: Thu, 3 Apr 2025 16:18:11 +0200 Subject: [PATCH 4/7] use gunicorn instead of flask dev server thx meme-lord? --- docker-compose.yml | 1 + services/api/requirements.txt | 1 + services/api/wsgi.py | 4 ++++ 3 files changed, 6 insertions(+) create mode 100644 services/api/wsgi.py diff --git a/docker-compose.yml b/docker-compose.yml index 842dc92e..218d5c30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,7 @@ services: - internal volumes: - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro + command: "gunicorn -w 3 -t 60 --log-level debug --capture-output --enable-stdio-inheritance -b 0.0.0.0:5000 webservice:application" environment: TULIP_MONGO: ${TULIP_MONGO} TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER} diff --git a/services/api/requirements.txt b/services/api/requirements.txt index 3f28156d..5676adc7 100644 --- a/services/api/requirements.txt +++ b/services/api/requirements.txt @@ -2,3 +2,4 @@ Flask_Cors pymongo Flask requests +gunicorn \ No newline at end of file diff --git a/services/api/wsgi.py b/services/api/wsgi.py new file mode 100644 index 00000000..6c9633dd --- /dev/null +++ b/services/api/wsgi.py @@ -0,0 +1,4 @@ +from webservice import application + +if __name__ == "__main__": + application.run(host='0.0.0.0', threaded=True) \ No newline at end of file From 48df5aa999a2800191345bec0de7ec2a2e2fc34f Mon Sep 17 00:00:00 2001 From: Lorenzo Leonardini Date: Thu, 3 Apr 2025 16:20:00 +0200 Subject: [PATCH 5/7] use nginx insteand of vite preview --- frontend/.dockerignore | 4 +++- frontend/Dockerfile-frontend | 9 +++++--- frontend/nginx.conf | 45 ++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 frontend/nginx.conf diff --git a/frontend/.dockerignore b/frontend/.dockerignore index b512c09d..75208534 100644 --- a/frontend/.dockerignore +++ b/frontend/.dockerignore @@ -1 +1,3 @@ -node_modules \ No newline at end of file +node_modules +dist +Dockerfile-frontend \ No newline at end of file diff --git a/frontend/Dockerfile-frontend b/frontend/Dockerfile-frontend index fbfc137b..55b2e7a4 100644 --- a/frontend/Dockerfile-frontend +++ b/frontend/Dockerfile-frontend @@ -1,4 +1,4 @@ -FROM node:17 +FROM node:17 AS build COPY package.json /app/ COPY yarn.lock /app/ @@ -11,6 +11,9 @@ COPY . /app/ RUN yarn run build -EXPOSE 3000 +FROM nginx + +COPY --from=build /app/dist /usr/share/nginx/html +COPY nginx.conf /etc/nginx/conf.d/default.conf -CMD yarn run preview --host --port 3000 +EXPOSE 3000 diff --git a/frontend/nginx.conf b/frontend/nginx.conf new file mode 100644 index 00000000..912601dd --- /dev/null +++ b/frontend/nginx.conf @@ -0,0 +1,45 @@ +server { + listen 3000; + + keepalive_timeout 1d; + send_timeout 1d; + client_body_timeout 1d; + client_header_timeout 1d; + proxy_connect_timeout 1d; + proxy_read_timeout 1d; + proxy_send_timeout 1d; + fastcgi_connect_timeout 1d; + fastcgi_read_timeout 1d; + fastcgi_send_timeout 1d; + memcached_connect_timeout 1d; + memcached_read_timeout 1d; + memcached_send_timeout 1d; + + gzip on; + gzip_types text/html application/javascript application/json text/css; + + # where the root here + root /usr/share/nginx/html; + # what file to server as index + index index.html; + + location /api/ { + proxy_pass http://api:5000/; + } + + location / { + # First attempt to serve request as file, then + # as directory, then fall back to redirecting to index.html + try_files $uri $uri/ $uri.html /index.html; + } + + location ~* \.(?:css|js|jpg|svg)$ { + expires 30d; + add_header Cache-Control "public"; + } + + location ~* \.(?:json)$ { + expires 1d; + add_header Cache-Control "public"; + } +} \ No newline at end of file From 6f5e66a5f399bee771c0ead33c10bf9891fd7009 Mon Sep 17 00:00:00 2001 From: Lorenzo Leonardini Date: Mon, 6 Apr 2026 18:25:31 +0200 Subject: [PATCH 6/7] Remove api command from dockerfile --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 761f7411..f3a731bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,6 @@ services: - internal volumes: - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro - command: "gunicorn -w 3 -t 60 --log-level debug --capture-output --enable-stdio-inheritance -b 0.0.0.0:5000 webservice:application" environment: TIMESCALE: ${TIMESCALE} TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER} From 90b37720d482e803808b1e4557eefc1209bd06fd Mon Sep 17 00:00:00 2001 From: Lorenzo Leonardini Date: Mon, 6 Apr 2026 18:40:03 +0200 Subject: [PATCH 7/7] update timescale clang to 19 --- services/timescale/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/timescale/Dockerfile b/services/timescale/Dockerfile index f5714e84..6ae461b0 100644 --- a/services/timescale/Dockerfile +++ b/services/timescale/Dockerfile @@ -1,6 +1,6 @@ FROM timescale/timescaledb:latest-pg15 -RUN apk add build-base make clang15 llvm15 git +RUN apk add build-base make clang19 llvm19 git COPY tulip /tulip RUN cd /tulip && make USE_PGXS=1 install && cd / && \ git clone https://github.com/ossc-db/pg_hint_plan --branch PG15 && \