Skip to content

[Shropshire] Job fetching filtering #473

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dracos wants to merge 4 commits into
base: master
Choose a base branch
from
Open

[Shropshire] Job fetching filtering #473

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
678f8ee
[Confirm] Add ability to ignore states in job fetching.
dracos Feb 12, 2026
97100aa
[Confirm] [Jobs] Switch to currentStatusLog.
dracos Feb 12, 2026
8784ef6
[Confirm] [Jobs] Refactor out some filtering.
dracos Feb 12, 2026
af5cbf2
[Shropshire] Add filtering for fetched jobs.
dracos Feb 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 15 additions & 43 deletions perllib/Integrations/Confirm.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -456,34 +456,19 @@ sub perform_request_graphql {
sub job_status_logs_graphql_query {
my ( $self, %args ) = @_;

my @job_type_codes
= keys %{ $self->config->{job_service_whitelist} // () };

my @status_codes
= keys %{ $self->config->{job_reverse_status_mapping} // () };

my (
$start_date,
$end_date,
$job_type_codes_str,
$status_codes_str,
) = (
$args{start_date},
$args{end_date},
join( '","', @job_type_codes ),
join( '","', @status_codes ),
);
my $job_type_codes = join '","', sort keys %{ $self->config->{job_service_whitelist} // () };
my $status_filter = $args{jobs_status_filter} || "";

return <<GRAPHQL;
{
jobStatusLogs(
filter: {
loggedDate: {
greaterThanEquals: "$start_date"
lessThanEquals: "$end_date"
greaterThanEquals: "$args{start_date}"
lessThanEquals: "$args{end_date}"
}
statusCode: {
inList: [ "$status_codes_str" ]
$status_filter
}
}
) {
Expand All @@ -496,7 +481,7 @@ sub job_status_logs_graphql_query {
jobType(
filter: {
code: {
inList: [ "$job_type_codes_str" ]
inList: [ "$job_type_codes" ]
}
}
){
Expand All @@ -511,49 +496,36 @@ GRAPHQL
sub jobs_graphql_query {
my ( $self, %args ) = @_;

my @job_type_codes
= keys %{ $self->config->{job_service_whitelist} // () };

my @status_codes
= keys %{ $self->config->{job_reverse_status_mapping} // () };

my (
$start_date,
$end_date,
$job_type_codes_str,
$status_codes_str,
) = (
$args{start_date},
$args{end_date},
join( '","', @job_type_codes ),
join( '","', @status_codes ),
);
my $job_type_codes = join '","', sort keys %{ $self->config->{job_service_whitelist} // () };
my $jobs_extra_filter = $args{jobs_extra_filter} || "";
my $status_filter = $args{jobs_status_filter} || "";

return <<"GRAPHQL"
{
jobs (
filter: {
entryDate: {
greaterThanEquals: "$start_date"
lessThanEquals: "$end_date"
greaterThanEquals: "$args{start_date}"
lessThanEquals: "$args{end_date}"
}
$jobs_extra_filter
}
){
jobType(
filter: {
code: {
inList: [ "$job_type_codes_str" ]
inList: [ "$job_type_codes" ]
}
}
){
code
name
}

statusLogs (
currentStatusLog (
Comment thread
dracos marked this conversation as resolved.
filter: {
statusCode: {
inList: [ "$status_codes_str" ]
$status_filter
}
}
) {
Expand Down
16 changes: 14 additions & 2 deletions perllib/Open311/Endpoint/Integration/Confirm.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -767,11 +767,13 @@ sub _get_service_request_updates_for_jobs {
my $status_logs = $integ->GetJobStatusLogs(
start_date => $args->{start_date},
end_date => $args->{end_date},
jobs_status_filter => $self->jobs_status_filter,
);

for my $log ( @{$status_logs} ) {
my $status
= $self->job_reverse_status_mapping->{ $log->{statusCode} };
next if $status && $status eq 'IGNORE';

if (!$status) {
# This shouldn't happen given that we filter by status code
Expand Down Expand Up @@ -1411,6 +1413,13 @@ and appends them to the $requests array as Open311 ServiceRequests.

=cut

sub jobs_extra_filter { "" }
sub jobs_status_filter {
my $self = shift;
my $status_codes = join '","', sort keys %{ $self->job_reverse_status_mapping // () };
return "inList: [ \"$status_codes\" ]";
}

sub _get_service_requests_for_jobs {
my ($self, $integ, $services, $args, $requests) = @_;

Expand All @@ -1419,6 +1428,8 @@ sub _get_service_requests_for_jobs {
my $jobs = $integ->GetJobs(
start_date => $args->{start_date},
end_date => $args->{end_date},
jobs_extra_filter => $self->jobs_extra_filter,
jobs_status_filter => $self->jobs_status_filter,
);

for my $job (@$jobs) {
Expand Down Expand Up @@ -1447,8 +1458,8 @@ sub _get_service_requests_for_jobs {
next;
}

my $last_status_log = $job->{statusLogs}[-1];
unless ($last_status_log) {
my $last_status_log = $job->{currentStatusLog};
unless ($last_status_log && %$last_status_log) {
$self->logger->warn( "no status logs for job type code "
. $job->{jobType}{code}
. " for job $job_id" );
Expand All @@ -1457,6 +1468,7 @@ sub _get_service_requests_for_jobs {

my $status = $self->job_reverse_status_mapping
->{ $last_status_log->{statusCode} };
next if $status && $status eq 'IGNORE';
unless ($status) {
# This shouldn't happen given that we filter by status code
# in graphql. But just in case, default to open.
Expand Down
12 changes: 12 additions & 0 deletions perllib/Open311/Endpoint/Integration/UK/Shropshire.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,16 @@ around process_service_request_args => sub {
return $ret;
};

sub jobs_extra_filter {
return <<"GRAPHQL";
contractCode: { equals: "ZVC18" }
statusFlag: { equals: "C" }
actualCompletionDate: { hasValue: false }
GRAPHQL
}

sub jobs_status_filter {
return 'notEquals: "X020"';
}

1;
42 changes: 14 additions & 28 deletions t/open311/endpoint/confirm.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,12 +318,10 @@ $open311->mock( perform_request_graphql => sub {
code => 'ASAP',
name => 'ASAP',
},
statusLogs => [
{
loggedDate => '2023-12-01T00:00:00',
statusCode => 'OPEN',
},
],
currentStatusLog => {
loggedDate => '2023-12-01T00:00:00',
statusCode => 'OPEN',
},
},
{
description => 'A completed job',
Expand All @@ -340,16 +338,10 @@ $open311->mock( perform_request_graphql => sub {
code => 'ASAP',
name => 'ASAP',
},
statusLogs => [
{
loggedDate => '2023-12-01T00:00:00',
statusCode => 'OPEN',
},
{
loggedDate => '2024-01-01T00:00:00',
statusCode => 'FIXED',
},
],
currentStatusLog => {
loggedDate => '2024-01-01T00:00:00',
statusCode => 'FIXED',
},
},

# Filtered out
Expand All @@ -368,16 +360,10 @@ $open311->mock( perform_request_graphql => sub {
code => 'ASAP',
name => 'ASAP',
},
statusLogs => [
{
loggedDate => '2023-12-01T00:00:00',
statusCode => 'OPEN',
},
{
loggedDate => '2023-12-01T01:00:00',
statusCode => 'SHUT',
},
],
currentStatusLog => {
loggedDate => '2023-12-01T01:00:00',
statusCode => 'SHUT',
},
},
{
description => 'A job with no status logs',
Expand All @@ -394,7 +380,7 @@ $open311->mock( perform_request_graphql => sub {
code => 'ASAP',
name => 'ASAP',
},
statusLogs => [],
currentStatusLog => {},
},
{
description => 'A job with EOFY priority',
Expand All @@ -411,7 +397,7 @@ $open311->mock( perform_request_graphql => sub {
code => 'EOFY',
name => 'End Of Financial Year',
},
statusLogs => [],
currentStatusLog => {},
},
],
},
Expand Down