diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48f7b314b..eb8dd3d69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: db: [sqlite, mysql, postgresql] - perl: ['5.40'] + perl: ['5.42'] include: - db: sqlite perl: '5.36' @@ -39,6 +39,9 @@ jobs: with: perl-version: ${{ matrix.perl }} + - name: update + run: sudo apt update -y + - name: Install binary dependencies run: | # * These were taken from the installation instruction. @@ -136,3 +139,41 @@ jobs: env: ZONEMASTER_BACKEND_CONFIG_FILE: ./share/backend_config.ci_${{ matrix.db }}.ini run: make test TEST_VERBOSE=1 + + build-artifact: + needs: run-tests + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - name: update + run: sudo apt update -y + + - name: apt install + run: sudo apt-get install -y build-essential git libmodule-install-perl gettext + + - name: build + run: perl Makefile.PL && make all dist + + - name: Get short SHA + id: short_sha + run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then + echo "SHORT_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-7)" >> $GITHUB_ENV + else + echo "SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV + fi + + - name: Get Zonemaster-Backend version + id: version + run: | + result=`grep "our $VERSION" lib/Zonemaster/Backend.pm` + result+='printf $VERSION;' + VERSION=`perl -e "$result"` + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: upload artifact + uses: actions/upload-artifact@v4 + with: + name: Zonemaster-Backend-${{ steps.version.outputs.version }}-${{ env.SHORT_SHA }} + path: Zonemaster-Backend-${{ steps.version.outputs.version }}.tar.gz diff --git a/MANIFEST b/MANIFEST index d61b87c60..32ae027ad 100644 --- a/MANIFEST +++ b/MANIFEST @@ -26,6 +26,7 @@ lib/Zonemaster/Backend/Log.pm lib/Zonemaster/Backend/Metrics.pm lib/Zonemaster/Backend/RPCAPI.pm lib/Zonemaster/Backend/TestAgent.pm +lib/Zonemaster/Backend/TLD_URL.pm lib/Zonemaster/Backend/Translator.pm lib/Zonemaster/Backend/Validator.pm LICENSE @@ -52,6 +53,7 @@ share/locale/nb/LC_MESSAGES/Zonemaster-Backend.mo share/locale/sl/LC_MESSAGES/Zonemaster-Backend.mo share/locale/sv/LC_MESSAGES/Zonemaster-Backend.mo share/Makefile +share/patch/patch_db_schema_version_1.pl share/patch/patch_db_zonemaster_backend_ver_11.1.0.pl share/patch/patch_db_zonemaster_backend_ver_11.2.0.pl share/patch/patch_db_zonemaster_backend_ver_9.0.0.pl @@ -76,9 +78,11 @@ t/batches.t t/config.t t/db.t t/db_ddl.t +t/db_schema_version.t t/idn.data t/idn.t t/lifecycle.t +t/log.t t/parameters_validation.t t/queue.t t/rpc_validation.t diff --git a/MANIFEST.SKIP b/MANIFEST.SKIP index 6e9a2c3c2..9a0a66a62 100644 --- a/MANIFEST.SKIP +++ b/MANIFEST.SKIP @@ -5,8 +5,8 @@ ^.*\.log ^.*\.swp$ ^MANIFEST\.SKIP$ -^Dockerfile$ -^zonemaster_launch$ +^docker/Dockerfile$ +^docker/zonemaster_launch$ ^\.github/ ^docs/internal-documentation/ \.po$ diff --git a/Makefile.PL b/Makefile.PL index 7815b8f52..d50371ca6 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -35,14 +35,16 @@ requires 'Router::Simple::Declare' => 0, 'Starman' => 0, 'Try::Tiny' => 0.12, - 'Zonemaster::LDNS' => 5.000001, # v5.0.1 - 'Zonemaster::Engine' => 8.001000, # v8.1.0 + 'Zonemaster::LDNS' => 5.000001, # v5.0.1 + 'Zonemaster::Engine' => 8.001000, # v8.1.0 ; +test_requires 'Capture::Tiny'; test_requires 'DBD::SQLite' => 1.66; test_requires 'Test::Differences'; +test_requires 'Test::Fatal'; test_requires 'Test::Exception'; -test_requires 'Time::Local' => 1.26; +test_requires 'Time::Local' => 1.26; test_requires 'Test::NoWarnings' => 0; recommends 'DBD::mysql'; @@ -61,25 +63,24 @@ no_index directory => 'Doc'; # Make all platforms include inc/Module/Install/External.pm requires_external_bin 'find'; -if ($^O eq "freebsd") { +if ( $^O eq "freebsd" ) { requires_external_bin 'gmake'; -}; +} sub MY::postamble { my $text; - if ($^O eq "freebsd") { + if ( $^O eq "freebsd" ) { + # Make FreeBSD use gmake for share/Makefile - $text = 'GMAKE ?= "gmake"' . "\n" - . 'pure_all :: share/Makefile' . "\n" - . "\t" . 'cd share && $(GMAKE) all' . "\n"; - } else { - $text = 'pure_all :: share/Makefile' . "\n" - . "\t" . 'cd share && $(MAKE) all' . "\n"; - }; + $text = 'GMAKE ?= "gmake"' . "\n" . 'pure_all :: share/Makefile' . "\n" . "\t" . 'cd share && $(GMAKE) all' . "\n"; + } + else { + $text = 'pure_all :: share/Makefile' . "\n" . "\t" . 'cd share && $(MAKE) all' . "\n"; + } my $docker = <<'END_DOCKER'; docker-build: - docker build --tag zonemaster/backend:local --build-arg version=$(VERSION) . + docker build -f docker/Dockerfile --tag zonemaster/backend:local --build-arg version=$(VERSION) . docker-tag-version: docker tag zonemaster/backend:local zonemaster/backend:$(VERSION) @@ -90,7 +91,7 @@ docker-tag-latest: END_DOCKER return $text . $docker; -}; +} install_share; diff --git a/README.md b/README.md index d2a7552df..f9f912e0f 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,15 @@ The Zonemaster Backend documentation is split up into several documents: it through its life cycle, all using JSON-RPC calls to the *RPC API daemon*. * The [API] documentation describes the *RPC API daemon* inteface in detail. +## CI artifact + +A tarball (`Zonemaster-Backend-.tar.gz`) is built and uploaded as a GitHub Actions artifact on every push and pull request. This artifact can be useful for release testing and PR review. +To download it: +1. Go to the [Actions tab](https://github.com/zonemaster/zonemaster-backend/actions) of the repository. +2. Select a workflow run (e.g. for a specific PR or branch). +3. Scroll to the bottom of the run summary to the **Artifacts** section. +4. Download the artifact named `Zonemaster-Backend--`. +The artifact name includes the module version and the first 7 characters of the commit SHA. ## License diff --git a/Dockerfile b/docker/Dockerfile similarity index 97% rename from Dockerfile rename to docker/Dockerfile index ccc050d00..2c0e8b257 100644 --- a/Dockerfile +++ b/docker/Dockerfile @@ -14,6 +14,7 @@ RUN apk add --no-cache \ RUN apk add --no-cache \ jq \ + perl-capture-tiny \ perl-class-method-modifiers \ perl-config-inifiles \ perl-dbd-sqlite \ @@ -78,7 +79,7 @@ RUN install -v -m 755 -o zonemaster -g zonemaster -d /var/lib/zonemaster USER zonemaster RUN $(perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backend")')/create_db.pl USER zonemaster -COPY zonemaster_launch /usr/local/bin +COPY docker/zonemaster_launch /usr/local/bin USER root ARG S6_OVERLAY_VERSION=3.2.1.0 diff --git a/docker/zonemaster_launch b/docker/zonemaster_launch new file mode 100755 index 000000000..15db37a6b --- /dev/null +++ b/docker/zonemaster_launch @@ -0,0 +1,67 @@ +#!/bin/sh + +# zonemaster_launch - Docker entrypoint script for Zonemaster Backend +# +# This script serves as the entrypoint for the Zonemaster Backend Docker container. +# It acts as a command dispatcher that starts different Zonemaster components based +# on the first argument passed to the container. +# +# How it works: +# - The script reads the first argument ($1) and uses a case statement to determine +# which component to launch. +# - For CLI tools (cli, zmb, zmtest), it passes any additional arguments to the +# respective command and exits after completion. +# - For services (rpcapi, testagent), it starts the process in the foreground, +# keeping the container running. +# - The 'full' option executes /init (from s6-overlay) which starts both the +# RPC API and testagent services together. +# - If no valid argument is provided, it displays a help message with available options. +# +# Usage: +# docker run zonemaster/zonemaster-backend cli # Run CLI command +# docker run zonemaster/zonemaster-backend rpcapi # Start RPC API server +# docker run zonemaster/zonemaster-backend testagent # Start test agent +# docker run zonemaster/zonemaster-backend full # Start all services + +case $1 in + + cli) + shift 1 + zonemaster-cli $@ + ;; + + zmb) + shift 1 + zmb $@ + ;; + + zmtest) + shift 1 + zmtest $@ + ;; + + rpcapi) + /usr/local/bin/starman --listen=0.0.0.0:5000 --preload-app --user=zonemaster --group=zonemaster /usr/local/bin/zonemaster_backend_rpcapi.psgi + + ;; + + testagent) + /usr/local/bin/zonemaster_backend_testagent --user=zonemaster --group=zonemaster --logfile=- foreground + ;; + + full) + exec /init + ;; + *) + echo "'$1' is not a valid option. +Available options: + - cli : pass argument to zonemaster-cli + - full : start both rpcapi & testagent + - rpcapi : start RPC API server + - testagent : start test agent + - zmb : run zmb command + - zmtest : run zmtest command" + exit 1 + ;; + +esac; diff --git a/lib/Zonemaster/Backend/Config.pm b/lib/Zonemaster/Backend/Config.pm index 78eba7b8c..8ad3e01e2 100644 --- a/lib/Zonemaster/Backend/Config.pm +++ b/lib/Zonemaster/Backend/Config.pm @@ -9,8 +9,8 @@ use Carp qw( confess croak ); use Config::IniFiles; use Config; use File::ShareDir qw[dist_file]; -use File::Slurp qw( read_file ); -use Log::Any qw( $log ); +use File::Slurp qw( read_file ); +use Log::Any qw( $log ); use Readonly; use Zonemaster::Backend::Validator qw( :untaint ); use Zonemaster::Backend::DB; @@ -149,7 +149,7 @@ Construct a new Zonemaster::Backend::Config based on a given configuration. ); The configuration is interpreted according to the -L. +L. Returns a new Zonemaster::Backend::Config instance with its properties set to normalized and untainted values according to the given configuration with @@ -182,13 +182,14 @@ sub parse { my $obj = bless( {}, $class ); $obj->{_public_profiles} = {}; $obj->{_private_profiles} = {}; + $obj->{_tld_url_override} = {}; my $ini = Config::IniFiles->new( -file => \$text ) or die "Failed to parse config: " . join( '; ', @Config::IniFiles::errors ) . "\n"; my $get_and_clear = sub { # Read and clear a property from a Config::IniFiles object. my ( $section, $param ) = @_; - my ( $value, @extra ) = $ini->val( $section, $param ); + my ( $value, @extra ) = $ini->val( $section, $param ); if ( @extra ) { die "Property not unique: $section.$param\n"; } @@ -198,7 +199,7 @@ sub parse { # Validate section names { - my %sections = map { $_ => 1 } ( 'DB', 'MYSQL', 'POSTGRESQL', 'SQLITE', 'LANGUAGE', 'PUBLIC PROFILES', 'PRIVATE PROFILES', 'ZONEMASTER', 'METRICS', 'RPCAPI' ); + my %sections = map { $_ => 1 } ( 'DB', 'MYSQL', 'POSTGRESQL', 'SQLITE', 'LANGUAGE', 'PUBLIC PROFILES', 'PRIVATE PROFILES', 'ZONEMASTER', 'METRICS', 'RPCAPI', 'TLD URL SETTINGS', 'TLD URL OVERRIDE' ); for my $section ( $ini->Sections ) { if ( !exists $sections{$section} ) { die "config: unrecognized section: $section\n"; @@ -210,13 +211,16 @@ sub parse { $obj->_set_DB_polling_interval( '0.5' ); $obj->_set_MYSQL_port( '3306' ); $obj->_set_POSTGRESQL_port( '5432' ); + $obj->_set_TLD_URL_SETTINGS_enable_tld_url( 'true' ); + $obj->_set_TLD_URL_SETTINGS_lookup_timeout( '3' ); + $obj->_set_TLD_URL_SETTINGS_include_source( 'true' ); $obj->_set_ZONEMASTER_max_zonemaster_execution_time( '600' ); $obj->_set_ZONEMASTER_number_of_processes_for_frontend_testing( '20' ); $obj->_set_ZONEMASTER_number_of_processes_for_batch_testing( '20' ); $obj->_set_ZONEMASTER_lock_on_queue( '0' ); $obj->_set_ZONEMASTER_age_reuse_previous_test( '600' ); - $obj->_set_RPCAPI_enable_user_create( 'no' ); # experimental - $obj->_set_RPCAPI_enable_batch_create( 'yes' ); # experimental + $obj->_set_RPCAPI_enable_user_create( 'no' ); # experimental + $obj->_set_RPCAPI_enable_batch_create( 'yes' ); # experimental $obj->_set_RPCAPI_enable_add_api_user( 'no' ); $obj->_set_RPCAPI_enable_add_batch_job( 'yes' ); $obj->_set_locales( 'en_US' ); @@ -235,6 +239,7 @@ sub parse { # Check deprecated properties and assign fallback values my @warnings; + #currently no deprecation warnings # Assign property values (part 2/2) @@ -277,6 +282,15 @@ sub parse { if ( defined( my $value = $get_and_clear->( 'SQLITE', 'database_file' ) ) ) { $obj->_set_SQLITE_database_file( $value ); } + if ( defined( my $value = $get_and_clear->( 'TLD URL SETTINGS', 'enable_tld_url' ) ) ) { + $obj->_set_TLD_URL_SETTINGS_enable_tld_url( $value ); + } + if ( defined( my $value = $get_and_clear->( 'TLD URL SETTINGS', 'lookup_timeout' ) ) ) { + $obj->_set_TLD_URL_SETTINGS_lookup_timeout( $value ); + } + if ( defined( my $value = $get_and_clear->( 'TLD URL SETTINGS', 'include_source' ) ) ) { + $obj->_set_TLD_URL_SETTINGS_include_source( $value ); + } if ( defined( my $value = $get_and_clear->( 'ZONEMASTER', 'max_zonemaster_execution_time' ) ) ) { $obj->_set_ZONEMASTER_max_zonemaster_execution_time( $value ); } @@ -304,7 +318,8 @@ sub parse { } $obj->_set_RPCAPI_enable_add_api_user( $value ); $obj->_set_RPCAPI_enable_user_create( $value ); - } else { + } + else { if ( defined( my $value = $get_and_clear->( 'RPCAPI', 'enable_add_api_user' ) ) ) { $obj->_set_RPCAPI_enable_add_api_user( $value ); $obj->_set_RPCAPI_enable_user_create( $value ); @@ -316,7 +331,8 @@ sub parse { } $obj->_set_RPCAPI_enable_add_batch_job( $value ); $obj->_set_RPCAPI_enable_batch_create( $value ); - } else { + } + else { if ( defined( my $value = $get_and_clear->( 'RPCAPI', 'enable_add_batch_job' ) ) ) { $obj->_set_RPCAPI_enable_add_batch_job( $value ); $obj->_set_RPCAPI_enable_batch_create( $value ); @@ -336,6 +352,11 @@ sub parse { $obj->_add_private_profile( $name, $path ); } + for my $name ( $ini->Parameters( 'TLD URL OVERRIDE' ) ) { + my $path = $get_and_clear->( 'TLD URL OVERRIDE', $name ); + $obj->_add_tld_url_override( $name, $path ); + } + # Check required propertys (part 2/2) if ( $obj->DB_engine eq 'MySQL' ) { die "config: missing required property MYSQL.host (required when DB.engine = MySQL)\n" @@ -410,10 +431,9 @@ sub check_db { return _normalize_engine_type( $db ); } - =head2 DB_engine -Get the value of L. +Get the value of L. Returns one of C<"SQLite">, C<"PostgreSQL"> or C<"MySQL">. @@ -436,28 +456,28 @@ sub _set_DB_engine { =head2 DB_polling_interval -Get the value of L. +Get the value of L. Returns a number. =head2 MYSQL_database -Get the value of L. +Get the value of L. Returns a string. =head2 MYSQL_host -Get the value of L. +Get the value of L. Returns a string. =head2 MYSQL_port -Returns the L +Returns the L property from the loaded config. Returns a number. @@ -465,35 +485,35 @@ Returns a number. =head2 MYSQL_password -Get the value of L. +Get the value of L. Returns a string. =head2 MYSQL_user -Get the value of L. +Get the value of L. Returns a string. =head2 POSTGRESQL_database -Get the value of L. +Get the value of L. Returns a string. =head2 POSTGRESQL_host -Get the value of L. +Get the value of L. Returns a string. =head2 POSTGRESQL_port -Returns the L +Returns the L property from the loaded config. Returns a number. @@ -501,28 +521,28 @@ Returns a number. =head2 POSTGRESQL_password -Get the value of L. +Get the value of L. Returns a string. =head2 POSTGRESQL_user -Get the value of L. +Get the value of L. Returns a string. =head2 SQLITE_database_file -Get the value of L. +Get the value of L. Returns a string. =head2 LANGUAGE_locale -Get the value of L. +Get the value of L. Returns a mapping from two-letter locale tag prefixes to full locale tags. This is represented by a hash mapping prefix to full locale tag. @@ -537,7 +557,7 @@ E.g.: =head2 PUBLIC_PROFILES -Get the set of L. +Get the set of L. Returns a hash mapping profile names to profile paths. The profile names are normalized to lowercase. @@ -547,16 +567,45 @@ C means that the Zonemaster Engine default profile should be used. =head2 PRIVATE_PROFILES -Get the set of L. +Get the set of L. Returns a hash mapping profile names to profile paths. The profile names are normalized to lowercase. Profile paths are always strings (contrast with L). +=head2 TLD_URL_SETTINGS_enable_tld_url + +Get the value of L. + +Returns a boolean. + + +=head2 TLD_URL_SETTINGS_lookup_timeout + +Get the value of L. + +Returns a positive integer. + + +=head2 TLD_URL_SETTINGS_include_source + +Get the value of L. + +Returns a boolean. + + +=head2 TLD_URL_OVERRIDE + +Get the set of L. + +Returns a hash mapping TLD label (ASCII or A-label) to URL string or blocking policy. +The TLD label is normalized to lowercase. See the meaning of +L. + =head2 ZONEMASTER_max_zonemaster_execution_time -Get the value of L. +Get the value of L. Returns a number. @@ -564,7 +613,7 @@ Returns a number. =head2 ZONEMASTER_number_of_processes_for_frontend_testing Get the value of -L. +L. Returns a number. @@ -572,7 +621,7 @@ Returns a number. =head2 ZONEMASTER_number_of_processes_for_batch_testing Get the value of -L. +L. Returns a number. @@ -580,7 +629,7 @@ Returns a number. =head2 ZONEMASTER_lock_on_queue Get the value of -L. +L. Returns a number. @@ -588,7 +637,7 @@ Returns a number. =head2 ZONEMASTER_age_reuse_previous_test Get the value of -L. +L. Returns a number. @@ -596,7 +645,7 @@ Returns a number. =head2 METRICS_statsd_host Get the value of -L. +L. Returns a string. @@ -604,7 +653,7 @@ Returns a string. =head2 METRICS_statsd_port Get the value of -L. +L. Returns a number. @@ -613,7 +662,7 @@ Returns a number. Experimental. Get the value of -L. +L. Return 0 or 1 @@ -622,7 +671,7 @@ Return 0 or 1 Experimental. Get the value of -L. +L. Return 0 or 1 @@ -630,7 +679,7 @@ Return 0 or 1 =head2 RPCAPI_enable_add_api_user Get the value of -L. +L. Return 0 or 1 @@ -638,7 +687,7 @@ Return 0 or 1 =head2 RPCAPI_enable_add_batch_job Get the value of -L. +L. Return 0 or 1 @@ -660,6 +709,10 @@ sub SQLITE_database_file { return $_[0]->{_SQLITE sub LANGUAGE_locale { return %{ $_[0]->{_LANGUAGE_locale} }; } sub PUBLIC_PROFILES { return %{ $_[0]->{_public_profiles} }; } sub PRIVATE_PROFILES { return %{ $_[0]->{_private_profiles} }; } +sub TLD_URL_SETTINGS_enable_tld_url { return $_[0]->{_TLD_URL_SETTINGS_enable_tld_url}; } +sub TLD_URL_SETTINGS_lookup_timeout { return $_[0]->{_TLD_URL_SETTINGS_lookup_timeout}; } +sub TLD_URL_SETTINGS_include_source { return $_[0]->{_TLD_URL_SETTINGS_include_source}; } +sub TLD_URL_OVERRIDE { return %{ $_[0]->{_tld_url_override} }; } sub ZONEMASTER_max_zonemaster_execution_time { return $_[0]->{_ZONEMASTER_max_zonemaster_execution_time}; } sub ZONEMASTER_lock_on_queue { return $_[0]->{_ZONEMASTER_lock_on_queue}; } sub ZONEMASTER_number_of_processes_for_frontend_testing { return $_[0]->{_ZONEMASTER_number_of_processes_for_frontend_testing}; } @@ -667,8 +720,8 @@ sub ZONEMASTER_number_of_processes_for_batch_testing { return $_[0]->{_ZONEMA sub ZONEMASTER_age_reuse_previous_test { return $_[0]->{_ZONEMASTER_age_reuse_previous_test}; } sub METRICS_statsd_host { return $_[0]->{_METRICS_statsd_host}; } sub METRICS_statsd_port { return $_[0]->{_METRICS_statsd_port}; } -sub RPCAPI_enable_user_create { return $_[0]->{_RPCAPI_enable_user_create}; } # experimental -sub RPCAPI_enable_batch_create { return $_[0]->{_RPCAPI_enable_batch_create}; } # experimental +sub RPCAPI_enable_user_create { return $_[0]->{_RPCAPI_enable_user_create}; } # experimental +sub RPCAPI_enable_batch_create { return $_[0]->{_RPCAPI_enable_batch_create}; } # experimental sub RPCAPI_enable_add_api_user { return $_[0]->{_RPCAPI_enable_add_api_user}; } sub RPCAPI_enable_add_batch_job { return $_[0]->{_RPCAPI_enable_add_batch_job}; } @@ -686,6 +739,9 @@ UNITCHECK { _create_setter( '_set_POSTGRESQL_password', '_POSTGRESQL_password', \&untaint_password ); _create_setter( '_set_POSTGRESQL_database', '_POSTGRESQL_database', \&untaint_postgresql_ident ); _create_setter( '_set_SQLITE_database_file', '_SQLITE_database_file', \&untaint_abs_path ); + _create_setter( '_set_TLD_URL_SETTINGS_enable_tld_url', '_TLD_URL_SETTINGS_enable_tld_url', \&untaint_json_bool ); + _create_setter( '_set_TLD_URL_SETTINGS_lookup_timeout', '_TLD_URL_SETTINGS_lookup_timeout', \&untaint_strictly_positive_int ); + _create_setter( '_set_TLD_URL_SETTINGS_include_source', '_TLD_URL_SETTINGS_include_source', \&untaint_json_bool ); _create_setter( '_set_ZONEMASTER_max_zonemaster_execution_time', '_ZONEMASTER_max_zonemaster_execution_time', \&untaint_strictly_positive_int ); _create_setter( '_set_ZONEMASTER_lock_on_queue', '_ZONEMASTER_lock_on_queue', \&untaint_non_negative_int ); _create_setter( '_set_ZONEMASTER_number_of_processes_for_frontend_testing', '_ZONEMASTER_number_of_processes_for_frontend_testing', \&untaint_strictly_positive_int ); @@ -693,34 +749,39 @@ UNITCHECK { _create_setter( '_set_ZONEMASTER_age_reuse_previous_test', '_ZONEMASTER_age_reuse_previous_test', \&untaint_strictly_positive_int ); _create_setter( '_set_METRICS_statsd_host', '_METRICS_statsd_host', \&untaint_host ); _create_setter( '_set_METRICS_statsd_port', '_METRICS_statsd_port', \&untaint_strictly_positive_int ); - _create_setter( '_set_RPCAPI_enable_user_create', '_RPCAPI_enable_user_create', \&untaint_bool ); # experimental - _create_setter( '_set_RPCAPI_enable_batch_create', '_RPCAPI_enable_batch_create', \&untaint_bool ); # experimental + _create_setter( '_set_RPCAPI_enable_user_create', '_RPCAPI_enable_user_create', \&untaint_bool ); # experimental + _create_setter( '_set_RPCAPI_enable_batch_create', '_RPCAPI_enable_batch_create', \&untaint_bool ); # experimental _create_setter( '_set_RPCAPI_enable_add_api_user', '_RPCAPI_enable_add_api_user', \&untaint_bool ); _create_setter( '_set_RPCAPI_enable_add_batch_job', '_RPCAPI_enable_add_batch_job', \&untaint_bool ); } -=head2 new_DB +=head2 new_DB( %opts ) Create a new database adapter object according to configuration. -The adapter connects to the database before it is returned. +=head3 INPUTS -=head3 INPUT +Options: + +=over 4 + +=item override_dbtype -The database adapter class is selected based on the return value of -L. -The database adapter class constructor is called without arguments and is -expected to configure itself according to available global configuration. +A L value. +Determines the database adapter class to invoke. +If not provided, the value of DB.engine is used. + +=back =head3 RETURNS -A configured L object. +A configured and connected L object. =head3 EXCEPTIONS =over 4 -=item Dies if no adapter for the configured database engine can be loaded. +=item Dies if an invalid option is given. =item Dies if the adapter is unable to connect to the database. @@ -729,9 +790,15 @@ A configured L object. =cut sub new_DB { - my ( $self ) = @_; + my ( $self, %opts ) = @_; + + my $dbtype_opt = delete $opts{override_dbtype}; + + if ( %opts ) { + croak 'Unrecognized options: ' . join( ', ', sort keys %opts ); + } - my $dbtype = $self->DB_engine; + my $dbtype = $self->check_db( $dbtype_opt // $self->DB_engine ); my $dbclass = Zonemaster::Backend::DB->get_db_class( $dbtype ); my $db = $dbclass->from_config( $self ); @@ -888,6 +955,28 @@ sub _add_private_profile { return; } +sub _add_tld_url_override { + my ( $self, $tld, $value ) = @_; + + unless ( untaint_tld_label( $tld ) ) { + die "Invalid TLD label in TLD URL OVERRIDE section: $tld\n"; + } + + if ( exists $self->{_tld_url_override}{$tld} ) { + die "TLD label not unique: $tld\n"; + } + + unless ( untaint_tld_block( $value ) + or untaint_tld_url_no_path( $value ) + or untaint_tld_url_string( $value ) ) + { + die "Invalid value for a TLD label key: $value\n"; + } + + $self->{_tld_url_override}{$tld} = $value; + return; +} + # Create a setter method with a given name using the given field and validator sub _create_setter { my ( $setter, $field, $validate ) = @_; diff --git a/lib/Zonemaster/Backend/DB.pm b/lib/Zonemaster/Backend/DB.pm index b83b6469f..c2376f35e 100644 --- a/lib/Zonemaster/Backend/DB.pm +++ b/lib/Zonemaster/Backend/DB.pm @@ -6,8 +6,9 @@ use Moose::Role; use 5.14.2; -use DBI qw(:sql_types); -use Digest::MD5 qw(md5_hex); +use Carp qw( croak ); +use DBI qw( :sql_types ); +use Digest::MD5 qw( md5_hex ); use Encode; use Exporter qw( import ); use JSON::PP; @@ -28,6 +29,7 @@ requires qw( get_dbh_specific_attributes get_relative_start_time is_duplicate + is_unknown_table ); has 'data_source_name' => ( @@ -54,6 +56,14 @@ has 'dbhandle' => ( required => 1, ); +=head2 $REQUIRED_SCHEMA_VERSION + +A positive integer. The database schema version that this module is compatible with. + +=cut + +Readonly our $REQUIRED_SCHEMA_VERSION => 1; + =head2 $TEST_WAITING The test is waiting to be processed. @@ -156,6 +166,66 @@ sub dbh { return $self->dbhandle; } +=head2 get_schema_version + +Detect the schema version of the database. + +Returns an unsigned integer. + +The C<0> value means the database is in a schema state prior to schema versioning. + +=cut + +sub get_schema_version { + my ( $self ) = @_; + + my $dbh = $self->dbh; + + local $dbh->{RaiseError} = 0; + local $dbh->{PrintError} = 0; + my $result = $dbh->selectcol_arrayref( "SELECT version FROM schema_version LIMIT 2" ); + + if ( $dbh->err ) { + if ( !$self->is_unknown_table ) { + croak "Failed to read schema version: " . $dbh->errstr; + } + + return 0; + } + + if ( @$result != 1 + || !defined $result->[0] + || $result->[0] !~ qr{^[1-9][0-9]*$} + || $result->[0] < 1 ) + { + croak 'Invalid schema version declaration'; + } + + return $result->[0]; +} + +=head2 assert_compatible_schema + +Assert that the database has a schema version that is compatible with this version of +Zonemaster Backend. + +Croaks if the database has an incompatible schema version, or if it is in an illegal state +with regard to schema version encoding. + +=cut + +sub assert_compatible_schema { + my ( $self ) = @_; + + my $db_schema_version = $self->get_schema_version; + + if ( $db_schema_version ne $REQUIRED_SCHEMA_VERSION ) { + croak "Expected schema version $REQUIRED_SCHEMA_VERSION, found $db_schema_version"; + } + + return; +} + sub add_api_user { my ( $self, $username, $api_key ) = @_; diff --git a/lib/Zonemaster/Backend/DB/MySQL.pm b/lib/Zonemaster/Backend/DB/MySQL.pm index 86ec1d7df..cbd12072b 100644 --- a/lib/Zonemaster/Backend/DB/MySQL.pm +++ b/lib/Zonemaster/Backend/DB/MySQL.pm @@ -205,6 +205,26 @@ sub create_schema { ' ) or die Zonemaster::Backend::Error::Internal->new( reason => "MySQL error, could not create 'users' table", data => $dbh->errstr() ); + #################################################################### + # SCHEMA VERSION + #################################################################### + $dbh->do( + "CREATE TABLE IF NOT EXISTS schema_version ( + id INTEGER PRIMARY KEY, + version INTEGER NOT NULL, + CHECK (id = 1), + CHECK (version >= 1) + )" + ) + or die Zonemaster::Backend::Error::Internal->new( + reason => "Failed to create 'schema_version' table", + data => $dbh->errstr() + ); + + if ( !defined $dbh->selectrow_array( "SELECT 1 FROM schema_version LIMIT 1" ) ) { + $dbh->do( "INSERT INTO schema_version (id, version) VALUES (1, ?)", {}, $Zonemaster::Backend::DB::REQUIRED_SCHEMA_VERSION ); + } + return; } @@ -234,6 +254,7 @@ sub drop_tables { } } + $self->dbh->do( "DROP TABLE IF EXISTS schema_version" ); $self->dbh->do( "DROP TABLE IF EXISTS test_results" ); $self->dbh->do( "DROP TABLE IF EXISTS result_entries" ); $self->dbh->do( "DROP TABLE IF EXISTS log_level" ); @@ -338,6 +359,15 @@ sub is_duplicate { return ( $self->dbh->err == 1062 ); } +sub is_unknown_table { + my ( $self ) = @_; + + # for the list of codes see: + # https://mariadb.com/kb/en/mariadb-error-codes/ + # https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html + return ( $self->dbh->err == 1146 ); +} + no Moose; __PACKAGE__->meta()->make_immutable(); diff --git a/lib/Zonemaster/Backend/DB/PostgreSQL.pm b/lib/Zonemaster/Backend/DB/PostgreSQL.pm index 617d12abc..581a310fa 100644 --- a/lib/Zonemaster/Backend/DB/PostgreSQL.pm +++ b/lib/Zonemaster/Backend/DB/PostgreSQL.pm @@ -184,6 +184,26 @@ sub create_schema { ' ) or die Zonemaster::Backend::Error::Internal->new( reason => "PostgreSQL error, could not create 'users' table", data => $dbh->errstr() ); + #################################################################### + # SCHEMA VERSION + #################################################################### + $dbh->do( + "CREATE TABLE IF NOT EXISTS schema_version ( + id INTEGER PRIMARY KEY, + version INTEGER NOT NULL, + CHECK (id = 1), + CHECK (version >= 1) + )" + ) + or die Zonemaster::Backend::Error::Internal->new( + reason => "Failed to create 'schema_version' table", + data => $dbh->errstr() + ); + + if ( !defined $dbh->selectrow_array( "SELECT 1 FROM schema_version LIMIT 1" ) ) { + $dbh->do( "INSERT INTO schema_version (id, version) VALUES (1, ?)", {}, $Zonemaster::Backend::DB::REQUIRED_SCHEMA_VERSION ); + } + return; } @@ -203,6 +223,7 @@ sub drop_tables { $self->dbh->do( "SET client_min_messages = warning" ); try { + $self->dbh->do( "DROP TABLE IF EXISTS schema_version CASCADE" ); $self->dbh->do( "DROP TABLE IF EXISTS test_results CASCADE" ); $self->dbh->do( "DROP TABLE IF EXISTS result_entries CASCADE" ); $self->dbh->do( "DROP TABLE IF EXISTS log_level" ); @@ -304,9 +325,23 @@ sub get_relative_start_time { sub is_duplicate { my ( $self ) = @_; + my $state = $self->dbh->state; + + # for the list of codes see: + # https://www.postgresql.org/docs/current/errcodes-appendix.html + return defined $state + && $state eq '23505'; +} + +sub is_unknown_table { + my ( $self ) = @_; + + my $state = $self->dbh->state; + # for the list of codes see: # https://www.postgresql.org/docs/current/errcodes-appendix.html - return ( $self->dbh->state == 23505 ); + return defined $state + && $state eq '42P01'; } no Moose; diff --git a/lib/Zonemaster/Backend/DB/SQLite.pm b/lib/Zonemaster/Backend/DB/SQLite.pm index e04108f89..95cd7cac9 100644 --- a/lib/Zonemaster/Backend/DB/SQLite.pm +++ b/lib/Zonemaster/Backend/DB/SQLite.pm @@ -183,6 +183,26 @@ sub create_schema { ' ) or die Zonemaster::Backend::Error::Internal->new( reason => "SQLite error, could not create 'users' table", data => $dbh->errstr() ); + #################################################################### + # SCHEMA VERSION + #################################################################### + $dbh->do( + "CREATE TABLE IF NOT EXISTS schema_version ( + id INTEGER PRIMARY KEY, + version INTEGER NOT NULL, + CHECK (id = 1), + CHECK (version >= 1) + )" + ) + or die Zonemaster::Backend::Error::Internal->new( + reason => "Failed to create 'schema_version' table", + data => $dbh->errstr() + ); + + if ( !defined $dbh->selectrow_array( "SELECT 1 FROM schema_version LIMIT 1" ) ) { + $dbh->do( "INSERT INTO schema_version (id, version) VALUES (1, ?)", {}, $Zonemaster::Backend::DB::REQUIRED_SCHEMA_VERSION ); + } + return; } @@ -195,8 +215,9 @@ Drop all the tables if they exist. sub drop_tables { my ( $self ) = @_; - $self->dbh->do( "DROP TABLE IF EXISTS test_results" ); + $self->dbh->do( "DROP TABLE IF EXISTS schema_version" ); $self->dbh->do( "DROP TABLE IF EXISTS result_entries" ); + $self->dbh->do( "DROP TABLE IF EXISTS test_results" ); $self->dbh->do( "DROP TABLE IF EXISTS log_level" ); $self->dbh->do( "DROP TABLE IF EXISTS users" ); $self->dbh->do( "DROP TABLE IF EXISTS batch_jobs" ); @@ -295,6 +316,14 @@ sub is_duplicate { return ( $self->dbh->err == 2067 ); } +sub is_unknown_table { + my ( $self ) = @_; + + # for the list of codes see: https://sqlite.org/rescode.html + return ( $self->dbh->err == 1 ) + && $self->dbh->errstr =~ /^no such table: /; +} + no Moose; __PACKAGE__->meta()->make_immutable(); diff --git a/lib/Zonemaster/Backend/Log.pm b/lib/Zonemaster/Backend/Log.pm index 52e30f339..26e9bce9c 100644 --- a/lib/Zonemaster/Backend/Log.pm +++ b/lib/Zonemaster/Backend/Log.pm @@ -13,16 +13,15 @@ use Data::Dumper; use base qw(Log::Any::Adapter::Base); - -my $default_level = Log::Any::Adapter::Util::numeric_level('info'); +my $default_level = Log::Any::Adapter::Util::numeric_level( 'info' ); sub init { - my ($self) = @_; + my ( $self ) = @_; if ( defined $self->{log_level} && $self->{log_level} =~ /\D/ ) { $self->{log_level} = lc $self->{log_level}; my $numeric_level = Log::Any::Adapter::Util::numeric_level( $self->{log_level} ); - if ( !defined($numeric_level) ) { + if ( !defined( $numeric_level ) ) { croak "Error: Unrecognized log level " . $self->{log_level} . "\n"; } $self->{log_level} = $numeric_level; @@ -31,54 +30,63 @@ sub init { $self->{log_level} //= $default_level; my $fd; - if ( !exists $self->{file} || $self->{file} eq '-') { + if ( !exists $self->{file} || $self->{file} eq '-' ) { if ( $self->{stderr} ) { - $fd = fileno(STDERR); - } else { - $fd = fileno(STDOUT); + $fd = fileno( STDERR ); + } + else { + $fd = fileno( STDOUT ); } - } else { + } + else { open( $fd, '>>', $self->{file} ) or croak "Can't open log file: $!"; } $self->{handle} = IO::Handle->new_from_fd( $fd, "w" ) or croak "Can't fdopen file: $!"; - $self->{handle}->autoflush(1); + $self->{handle}->autoflush( 1 ); + + $self->{with_timestamp} //= 1; + $self->{with_pid} //= 1; if ( !exists $self->{formatter} ) { if ( $self->{json} ) { $self->{formatter} = \&format_json; - } else { + } + else { $self->{formatter} = \&format_text; } } } sub format_text { - my ($self, $log_params) = @_; - my $msg; - $msg .= sprintf "%s ", $log_params->{timestamp}; - delete $log_params->{timestamp}; - $msg .= sprintf( - "[%d] [%s] [%s] %s", - delete $log_params->{pid}, - uc delete $log_params->{level}, - delete $log_params->{category}, - delete $log_params->{message} - ); + my ( $self, $log_params ) = @_; + my $msg = ''; + + my $timestamp = delete $log_params->{timestamp}; + if ( defined $timestamp ) { + $msg .= sprintf "%s ", $timestamp; + } + + my $pid = delete $log_params->{pid}; + if ( defined $pid ) { + $msg .= sprintf "[%d] ", $pid; + } + + $msg .= sprintf( "[%s] [%s] %s", uc delete $log_params->{level}, delete $log_params->{category}, delete $log_params->{message} ); if ( %$log_params ) { local $Data::Dumper::Indent = 0; - local $Data::Dumper::Terse = 1; - my $data = Dumper($log_params); + local $Data::Dumper::Terse = 1; + my $data = Dumper( $log_params ); $msg .= " Extra parameters: $data"; } - return $msg + return $msg; } sub format_json { - my ($self, $log_params) = @_; + my ( $self, $log_params ) = @_; my $js = JSON::PP->new; $js->canonical( 1 ); @@ -86,42 +94,106 @@ sub format_json { return $js->encode( $log_params ); } - sub structured { - my ($self, $level, $category, $string, @items) = @_; + my ( $self, $level, $category, $string, @items ) = @_; - my $log_level = Log::Any::Adapter::Util::numeric_level($level); + my $log_level = Log::Any::Adapter::Util::numeric_level( $level ); return if $log_level > $self->{log_level}; my %log_params = ( - timestamp => strftime( "%FT%TZ", gmtime ), - level => $level, + level => $level, category => $category, - message => $string, - pid => $PID, + message => $string, ); + if ( $self->{with_timestamp} ) { + $log_params{timestamp} = strftime( "%FT%TZ", gmtime ); + } + + if ( $self->{with_pid} ) { + $log_params{pid} = $PID; + } + for my $item ( @items ) { - if (ref($item) eq 'HASH') { - for my $key (keys %$item) { + if ( ref( $item ) eq 'HASH' ) { + for my $key ( keys %$item ) { $log_params{$key} = $item->{$key}; } } } - my $msg = $self->{formatter}->($self, \%log_params); - $self->{handle}->print($msg . "\n"); + my $msg = $self->{formatter}->( $self, \%log_params ); + $self->{handle}->print( $msg . "\n" ); } # From Log::Any::Adapter::File foreach my $method ( Log::Any::Adapter::Util::detection_methods() ) { no strict 'refs'; - my $base = substr($method,3); + my $base = substr( $method, 3 ); my $method_level = Log::Any::Adapter::Util::numeric_level( $base ); *{$method} = sub { - return !!( $method_level <= $_[0]->{log_level} ); + return !!( $method_level <= $_[0]->{log_level} ); }; } 1; + +=head1 NAME + +Zonemaster::Backend::Log + +=head1 SYNOPSIS + + Log::Any::Adapter->set( + '+Zonemaster::Backend::Log', + log_level => 'info', + json => 0, + file => '/path/to/logfile.log', + with_pid => 1, + with_timestamp => 1, + ); + +=head1 DESCRIPTION + +This is an adapter for Log::Any, tailored towards the needs of Zonemaster +Backend. + +The following attributes are supported. + +=over 4 + +=item file + +A string. The location of the log file to use. Default: C<->. + +The special value C<-> sends output to stdout or stderr depending on the +C attribute. + +=item stderr + +A boolean. True means log to stderr. False means log to stdout. Default: false. + +Ignored if C is anything other than C<->. + +=item log_level + +The threshold for emitting log entries. Default: info. + +The allowed values are specified at L. + +=item json + +A boolean. When true, logs are written in JSON format. Default: false. + +=item with_timestamp + +A boolean. Controls the inclusion of timestamp log entries. Default: true. + +=item with_pid + +A boolean. Controls the inclusion of PID in log entries. Default: true. + +=back + +=cut diff --git a/lib/Zonemaster/Backend/RPCAPI.pm b/lib/Zonemaster/Backend/RPCAPI.pm index e7e8f392f..00bb0cebc 100644 --- a/lib/Zonemaster/Backend/RPCAPI.pm +++ b/lib/Zonemaster/Backend/RPCAPI.pm @@ -1,35 +1,32 @@ package Zonemaster::Backend::RPCAPI; -use strict; -use warnings; use 5.14.2; +use warnings; -# Public Modules -use DBI qw(:utils); -use Digest::MD5 qw(md5_hex); -use File::Slurp qw(append_file); +use Carp qw( croak ); +use DBI qw( :utils ); +use Digest::MD5 qw( md5_hex ); +use Encode; +use File::Slurp qw( append_file ); use HTML::Entities; use JSON::PP; use JSON::Validator::Joi; -use Log::Any qw($log); -use Mojo::JSON::Pointer; -use Scalar::Util qw(blessed); use JSON::Validator::Schema::Draft7; -use Locale::TextDomain qw[Zonemaster-Backend]; -use Locale::Messages qw[LC_MESSAGES LC_ALL]; -use POSIX qw (setlocale); -use Encode; - -# Zonemaster Modules -use Zonemaster::Engine; +use Locale::Messages qw( LC_MESSAGES LC_ALL ); +use Locale::TextDomain qw( Zonemaster-Backend ); +use Log::Any qw( $log ); +use Mojo::JSON::Pointer; +use POSIX qw( setlocale ); +use Scalar::Util qw( blessed ); +use Zonemaster::Backend::Errors; +use Zonemaster::Backend::TLD_URL; +use Zonemaster::Backend::Translator; +use Zonemaster::Backend::Validator; +use Zonemaster::Backend; use Zonemaster::Engine::Normalization qw( normalize_name trim_space ); use Zonemaster::Engine::Profile; use Zonemaster::Engine::Recursor; -use Zonemaster::Backend; -use Zonemaster::Backend::Config; -use Zonemaster::Backend::Translator; -use Zonemaster::Backend::Validator; -use Zonemaster::Backend::Errors; +use Zonemaster::Engine; my $zm_validator = Zonemaster::Backend::Validator->new; our %json_schemas; @@ -40,51 +37,36 @@ sub joi { } sub new { - my ( $type, $params ) = @_; + my ( $class, %args ) = @_; - my $self = {}; - bless( $self, $type ); + my $config = delete $args{config} + or croak "Missing 'config' argument"; - if ( ! $params || ! $params->{config} ) { - handle_exception("Missing 'config' parameter"); - } + my $db = delete $args{db} + or croak "Missing 'db' argument"; - $self->{config} = $params->{config}; + my $profiles = delete $args{profiles} + or croak "Missing 'profiles' argument"; - my $dbtype; - if ( $params->{dbtype} ) { - $dbtype = $self->{config}->check_db($params->{dbtype}); - } else { - $dbtype = $self->{config}->DB_engine; + if ( %args ) { + croak 'Unrecognized arguments: ' . join( ', ', sort keys %args ); } - $self->_init_db($dbtype); + $db->assert_compatible_schema; - $self->{_profiles} = Zonemaster::Backend::Config->load_profiles( # - $self->{config}->PUBLIC_PROFILES, - $self->{config}->PRIVATE_PROFILES, - ); - - return ( $self ); -} - -sub _init_db { - my ( $self, $dbtype ) = @_; - - eval { - my $dbclass = Zonemaster::Backend::DB->get_db_class( $dbtype ); - $self->{db} = $dbclass->from_config( $self->{config} ); + my $obj = { + config => $config, + db => $db, + _profiles => $profiles, }; - if ($@) { - handle_exception("Failed to initialize the [$dbtype] database backend module: [$@]"); - } + return bless $obj, $class; } sub handle_exception { my ( $exception ) = @_; - if ( !$exception->isa('Zonemaster::Backend::Error') ) { + if ( !$exception->isa( 'Zonemaster::Backend::Error' ) ) { my $reason = $exception; $exception = Zonemaster::Backend::Error::Internal->new( reason => $reason ); } @@ -92,26 +74,28 @@ sub handle_exception { my $log_extra = $exception->as_hash; delete $log_extra->{message}; - if ( $exception->isa('Zonemaster::Backend::Error::Internal') ) { - $log->error($exception->as_string, $log_extra); - } else { - $log->info($exception->as_string, $log_extra); + if ( $exception->isa( 'Zonemaster::Backend::Error::Internal' ) ) { + $log->error( $exception->as_string, $log_extra ); + } + else { + $log->info( $exception->as_string, $log_extra ); } die $exception->as_hash; } $json_schemas{version_info} = joi->object->strict; + sub version_info { my ( $self ) = @_; my %ver; eval { - $ver{zonemaster_ldns} = Zonemaster::LDNS->VERSION; - $ver{zonemaster_engine} = Zonemaster::Engine->VERSION; + $ver{zonemaster_ldns} = Zonemaster::LDNS->VERSION; + $ver{zonemaster_engine} = Zonemaster::Engine->VERSION; $ver{zonemaster_backend} = Zonemaster::Backend->VERSION; }; - if ($@) { + if ( $@ ) { handle_exception( $@ ); } @@ -120,11 +104,13 @@ sub version_info { # Experimental $json_schemas{system_versions} = $json_schemas{version_info}; + sub system_versions { return version_info( @_ ); } $json_schemas{profile_names} = joi->object->strict; + sub profile_names { my ( $self ) = @_; @@ -139,16 +125,16 @@ sub profile_names { # Experimental $json_schemas{conf_profiles} = $json_schemas{profile_names}; + sub conf_profiles { - my $result = { - profiles => profile_names( @_ ) - }; + my $result = { profiles => profile_names( @_ ) }; return $result; } # Return the list of language tags supported by get_test_results(). The tags are # derived from the locale tags set in the configuration file. $json_schemas{get_language_tags} = joi->object->strict; + sub get_language_tags { my ( $self ) = @_; @@ -167,33 +153,60 @@ sub get_language_tags { # Experimental $json_schemas{conf_languages} = $json_schemas{get_language_tags}; + sub conf_languages { - my $result = { - languages => get_language_tags( @_ ) - }; + my $result = { languages => get_language_tags( @_ ) }; return $result; } +=head2 get_tld_url + +Handles the RPCAPI with the same name. All the "dirty work" is done in +a separate subroutine Zonemaster::Backend::TLD_URL::process in a +separate Perl module. + +=cut + +$json_schemas{get_tld_url} = { + type => 'object', + additionalProperties => 0, + required => ['domain'], + properties => { + domain => $zm_validator->domain_name + } +}; + +sub get_tld_url { + my ( $self, $params ) = @_; + my $domain; + ( undef, $domain ) = normalize_name( trim_space( $params->{domain} ) ); + + return Zonemaster::Backend::TLD_URL::process( $self, $domain ); +} + $json_schemas{get_host_by_name} = { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'hostname' ], - properties => { + required => ['hostname'], + properties => { hostname => $zm_validator->domain_name } }; + sub get_host_by_name { my ( $self, $params ) = @_; my @adresses; eval { - my $ns_name = $params->{hostname}; + my $ns_name = $params->{hostname}; - @adresses = map { {$ns_name => $_->short} } $recursor->get_addresses_for($ns_name); + @adresses = map { + { $ns_name => $_->short } + } $recursor->get_addresses_for( $ns_name ); @adresses = { $ns_name => '0.0.0.0' } if not @adresses; }; - if ($@) { + if ( $@ ) { handle_exception( $@ ); } @@ -202,46 +215,46 @@ sub get_host_by_name { # Experimental $json_schemas{lookup_address_records} = $json_schemas{get_host_by_name}; + sub lookup_address_records { - my $result = { - address_records => get_host_by_name( @_ ) - }; + my $result = { address_records => get_host_by_name( @_ ) }; return $result; } $json_schemas{get_data_from_parent_zone} = { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'domain' ], - properties => { - domain => $zm_validator->domain_name, + required => ['domain'], + properties => { + domain => $zm_validator->domain_name, language => $zm_validator->language_tag, } }; + sub get_data_from_parent_zone { my ( $self, $params ) = @_; my $result = eval { my %result; my $domain = $params->{domain}; - my ( $_errors, $normalized_domain ) = normalize_name( trim_space ( $domain ) ); + my ( $_errors, $normalized_domain ) = normalize_name( trim_space( $domain ) ); my @ns_list; my @ns_names; my $zone = Zonemaster::Engine->zone( $normalized_domain ); - push @ns_list, { ns => $_->name->string, ip => $_->address->short} for @{$zone->glue}; + push @ns_list, { ns => $_->name->string, ip => $_->address->short } for @{ $zone->glue }; my @ds_list; - $zone = Zonemaster::Engine->zone($normalized_domain); + $zone = Zonemaster::Engine->zone( $normalized_domain ); my $ds_p = $zone->parent->query_one( $zone->name, 'DS', { dnssec => 1, cd => 1, recurse => 1 } ); - if ($ds_p) { + if ( $ds_p ) { my @ds = $ds_p->get_records( 'DS', 'answer' ); foreach my $ds ( @ds ) { next unless $ds->type eq 'DS'; - push(@ds_list, { keytag => $ds->keytag, algorithm => $ds->algorithm, digtype => $ds->digtype, digest => $ds->hexdigest }); + push( @ds_list, { keytag => $ds->keytag, algorithm => $ds->algorithm, digtype => $ds->digtype, digest => $ds->hexdigest } ); } } @@ -249,45 +262,47 @@ sub get_data_from_parent_zone { $result{ds_list} = \@ds_list; return \%result; }; - if ($@) { + if ( $@ ) { handle_exception( $@ ); } - elsif ($result) { + elsif ( $result ) { return $result; } } # Experimental $json_schemas{lookup_delegation_data} = $json_schemas{get_data_from_parent_zone}; + sub lookup_delegation_data { return get_data_from_parent_zone( @_ ); } $json_schemas{start_domain_test} = { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'domain' ], - properties => { - domain => $zm_validator->domain_name, - ipv4 => joi->boolean->compile, - ipv6 => joi->boolean->compile, + required => ['domain'], + properties => { + domain => $zm_validator->domain_name, + ipv4 => joi->boolean->compile, + ipv6 => joi->boolean->compile, nameservers => { - type => 'array', + type => 'array', items => $zm_validator->nameserver }, ds_info => { - type => 'array', + type => 'array', items => $zm_validator->ds_info }, - profile => $zm_validator->profile_name, - client_id => $zm_validator->client_id->compile, + profile => $zm_validator->profile_name, + client_id => $zm_validator->client_id->compile, client_version => $zm_validator->client_version->compile, - config => joi->string->compile, - priority => $zm_validator->priority->compile, - queue => $zm_validator->queue->compile, - language => $zm_validator->language_tag, + config => joi->string->compile, + priority => $zm_validator->priority->compile, + queue => $zm_validator->queue->compile, + language => $zm_validator->language_tag, } }; + sub start_domain_test { my ( $self, $params ) = @_; @@ -303,7 +318,7 @@ sub start_domain_test { $result = $self->{db}->create_new_test( $params->{domain}, $params, $self->{config}->ZONEMASTER_age_reuse_previous_test ); }; - if ($@) { + if ( $@ ) { handle_exception( $@ ); } @@ -312,16 +327,16 @@ sub start_domain_test { # Experimental $json_schemas{job_create} = $json_schemas{start_domain_test}; + sub job_create { - my $result = { - job_id => start_domain_test( @_ ) - }; + my $result = { job_id => start_domain_test( @_ ) }; return $result; } -$json_schemas{test_progress} = joi->object->strict->props( +$json_schemas{test_progress} = joi->object->strict->props( # test_id => $zm_validator->test_id->required ); + sub test_progress { my ( $self, $params ) = @_; @@ -330,7 +345,7 @@ sub test_progress { my $test_id = $params->{test_id}; $result = $self->{db}->test_progress( $test_id ); }; - if ($@) { + if ( $@ ) { handle_exception( $@ ); } @@ -338,23 +353,23 @@ sub test_progress { } # Experimental -$json_schemas{job_status} = joi->object->strict->props( +$json_schemas{job_status} = joi->object->strict->props( # job_id => $zm_validator->test_id->required ); + sub job_status { my ( $self, $params ) = @_; $params->{test_id} = delete $params->{job_id}; - my $result = { - progress => $self->test_progress( $params ) - }; + my $result = { progress => $self->test_progress( $params ) }; return $result; } -$json_schemas{get_test_params} = joi->object->strict->props( +$json_schemas{get_test_params} = joi->object->strict->props( # test_id => $zm_validator->test_id->required ); + sub get_test_params { my ( $self, $params ) = @_; @@ -364,7 +379,7 @@ sub get_test_params { $result = $self->{db}->get_test_params( $test_id ); }; - if ($@) { + if ( $@ ) { handle_exception( $@ ); } @@ -372,9 +387,10 @@ sub get_test_params { } # Experimental -$json_schemas{job_params} = joi->object->strict->props( +$json_schemas{job_params} = joi->object->strict->props( # job_id => $zm_validator->test_id->required ); + sub job_params { my ( $self, $params ) = @_; @@ -384,19 +400,20 @@ sub job_params { } $json_schemas{get_test_results} = { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'id', 'language' ], - properties => { - id => $zm_validator->test_id->required->compile, + required => [ 'id', 'language' ], + properties => { + id => $zm_validator->test_id->required->compile, language => $zm_validator->language_tag, } }; + sub get_test_results { my ( $self, $params ) = @_; my $result; - eval{ + eval { my $locale = $self->_get_locale( $params ); @@ -408,7 +425,7 @@ sub get_test_results { die "Failed to set locale: $locale"; } - eval { $translator->data } if $translator; # Provoke lazy loading of translation data + eval { $translator->data } if $translator; # Provoke lazy loading of translation data my @zm_results; my %testcases; @@ -426,13 +443,13 @@ sub get_test_results { next; } - $res->{module} = $test_res->{module}; + $res->{module} = $test_res->{module}; $res->{message} = $translator->translate_tag( $test_res ) . "\n"; $res->{message} =~ s/,/, /isg; $res->{message} =~ s/;/; /isg; - $res->{level} = $test_res->{level}; - $res->{testcase} = $test_res->{testcase} // 'UNSPECIFIED'; - $testcases{$res->{testcase}} = $translator->test_case_description($res->{testcase}); + $res->{level} = $test_res->{level}; + $res->{testcase} = $test_res->{testcase} // 'UNSPECIFIED'; + $testcases{ $res->{testcase} } = $translator->test_case_description( $res->{testcase} ); if ( $test_res->{module} eq 'SYSTEM' ) { if ( $res->{message} =~ /policy\.json/ ) { @@ -462,16 +479,16 @@ sub get_test_results { push( @zm_results, $res ); } - $result = $test_info; + $result = $test_info; $result->{testcase_descriptions} = \%testcases; - $result->{results} = \@zm_results; + $result->{results} = \@zm_results; $translator->locale( $previous_locale ); $result = $test_info; $result->{results} = \@zm_results; }; - if ($@) { + if ( $@ ) { handle_exception( $@ ); } @@ -480,14 +497,15 @@ sub get_test_results { # Experimental $json_schemas{job_results} = { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'job_id', 'language' ], - properties => { - job_id => $zm_validator->test_id->required->compile, + required => [ 'job_id', 'language' ], + properties => { + job_id => $zm_validator->test_id->required->compile, language => $zm_validator->language_tag, } }; + sub job_results { my ( $self, $params ) = @_; @@ -505,23 +523,24 @@ sub job_results { } $json_schemas{get_test_history} = { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'frontend_params' ], - properties => { - offset => joi->integer->min(0)->compile, - limit => joi->integer->min(0)->compile, - filter => joi->string->regex('^(?:all|delegated|undelegated)$')->compile, + required => ['frontend_params'], + properties => { + offset => joi->integer->min( 0 )->compile, + limit => joi->integer->min( 0 )->compile, + filter => joi->string->regex( '^(?:all|delegated|undelegated)$' )->compile, frontend_params => { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'domain' ], - properties => { + required => ['domain'], + properties => { domain => $zm_validator->domain_name } } } }; + sub get_test_history { my ( $self, $params ) = @_; @@ -529,15 +548,17 @@ sub get_test_history { eval { $params->{offset} //= 0; - $params->{limit} //= 200; + $params->{limit} //= 200; $params->{filter} //= "all"; $results = $self->{db}->get_test_history( $params ); - my @results = map { { %$_, undelegated => $_->{undelegated} ? JSON::PP::true : JSON::PP::false } } @$results; + my @results = map { + { %$_, undelegated => $_->{undelegated} ? JSON::PP::true : JSON::PP::false } + } @$results; $results = \@results; }; - if ($@) { + if ( $@ ) { handle_exception( $@ ); } @@ -546,23 +567,24 @@ sub get_test_history { # Experimental $json_schemas{domain_history} = { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'params' ], - properties => { - offset => joi->integer->min(0)->compile, - limit => joi->integer->min(0)->compile, - filter => joi->string->regex('^(?:all|delegated|undelegated)$')->compile, + required => ['params'], + properties => { + offset => joi->integer->min( 0 )->compile, + limit => joi->integer->min( 0 )->compile, + filter => joi->string->regex( '^(?:all|delegated|undelegated)$' )->compile, params => { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'domain' ], - properties => { + required => ['domain'], + properties => { domain => $zm_validator->domain_name } } } }; + sub domain_history { my ( $self, $params ) = @_; @@ -573,7 +595,7 @@ sub domain_history { return { history => [ map { - { + { # job_id => $_->{id}, created_at => $_->{created_at}, overall_result => $_->{overall_result}, @@ -586,8 +608,9 @@ sub domain_history { $json_schemas{add_api_user} = joi->object->strict->props( username => $zm_validator->username->required, - api_key => $zm_validator->api_key->required, + api_key => $zm_validator->api_key->required, ); + sub add_api_user { my ( $self, $params, undef, $remote_ip ) = @_; @@ -608,11 +631,11 @@ sub add_api_user { else { die Zonemaster::Backend::Error::PermissionDenied->new( message => 'Call to "add_api_user" method not permitted from a remote IP', - data => { remote_ip => $remote_ip } + data => { remote_ip => $remote_ip } ); } }; - if ($@) { + if ( $@ ) { handle_exception( $@ ); } @@ -621,50 +644,50 @@ sub add_api_user { # Experimental $json_schemas{user_create} = $json_schemas{add_api_user}; + sub user_create { - my $result = { - success => add_api_user( @_ ) - }; + my $result = { success => add_api_user( @_ ) }; return $result; } $json_schemas{add_batch_job} = { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'username', 'api_key', 'domains' ], - properties => { + required => [ 'username', 'api_key', 'domains' ], + properties => { username => $zm_validator->username->required->compile, - api_key => $zm_validator->api_key->required->compile, - domains => { - type => "array", + api_key => $zm_validator->api_key->required->compile, + domains => { + type => "array", additionalItems => 0, - items => $zm_validator->domain_name, - minItems => 1 + items => $zm_validator->domain_name, + minItems => 1 }, test_params => { - type => 'object', + type => 'object', additionalProperties => 0, - properties => { - ipv4 => joi->boolean->compile, - ipv6 => joi->boolean->compile, + properties => { + ipv4 => joi->boolean->compile, + ipv6 => joi->boolean->compile, nameservers => { - type => 'array', + type => 'array', items => $zm_validator->nameserver }, ds_info => { - type => 'array', + type => 'array', items => $zm_validator->ds_info }, - profile => $zm_validator->profile_name, - client_id => $zm_validator->client_id->compile, + profile => $zm_validator->profile_name, + client_id => $zm_validator->client_id->compile, client_version => $zm_validator->client_version->compile, - config => joi->string->compile, - priority => $zm_validator->priority->compile, - queue => $zm_validator->queue->compile, + config => joi->string->compile, + priority => $zm_validator->priority->compile, + queue => $zm_validator->queue->compile, } } } }; + sub add_batch_job { my ( $self, $params ) = @_; @@ -680,7 +703,7 @@ sub add_batch_job { $results = $self->{db}->add_batch_job( $params ); }; - if ($@) { + if ( $@ ) { handle_exception( $@ ); } @@ -689,73 +712,71 @@ sub add_batch_job { # Experimental $json_schemas{batch_create} = { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'username', 'api_key', 'domains' ], - properties => { + required => [ 'username', 'api_key', 'domains' ], + properties => { username => $zm_validator->username->required->compile, - api_key => $zm_validator->api_key->required->compile, - domains => { - type => "array", + api_key => $zm_validator->api_key->required->compile, + domains => { + type => "array", additionalItems => 0, - items => $zm_validator->domain_name, - minItems => 1 + items => $zm_validator->domain_name, + minItems => 1 }, job_params => { - type => 'object', + type => 'object', additionalProperties => 0, - properties => { - ipv4 => joi->boolean->compile, - ipv6 => joi->boolean->compile, + properties => { + ipv4 => joi->boolean->compile, + ipv6 => joi->boolean->compile, nameservers => { - type => 'array', + type => 'array', items => $zm_validator->nameserver }, ds_info => { - type => 'array', + type => 'array', items => $zm_validator->ds_info }, - profile => $zm_validator->profile_name, - client_id => $zm_validator->client_id->compile, + profile => $zm_validator->profile_name, + client_id => $zm_validator->client_id->compile, client_version => $zm_validator->client_version->compile, - config => joi->string->compile, - priority => $zm_validator->priority->compile, - queue => $zm_validator->queue->compile, + config => joi->string->compile, + priority => $zm_validator->priority->compile, + queue => $zm_validator->queue->compile, } } } }; + sub batch_create { my ( $self, $params ) = @_; $params->{test_params} = delete $params->{job_params}; - my $result = { - batch_id => $self->add_batch_job( $params ) - }; + my $result = { batch_id => $self->add_batch_job( $params ) }; return $result; } $json_schemas{batch_status} = { - type => 'object', + type => 'object', additionalProperties => 0, - required => [ 'batch_id' ], - properties => { - batch_id => $zm_validator->batch_id->required, - list_waiting_tests => joi->boolean->compile, - list_running_tests => joi->boolean->compile, + required => ['batch_id'], + properties => { + batch_id => $zm_validator->batch_id->required, + list_waiting_tests => joi->boolean->compile, + list_running_tests => joi->boolean->compile, list_finished_tests => joi->boolean->compile, } }; + sub batch_status { my ( $self, $params ) = @_; my $result; - eval { - $result = $self->{db}->batch_status($params); - }; - if ($@) { + eval { $result = $self->{db}->batch_status( $params ); }; + if ( $@ ) { handle_exception( $@ ); } return $result; @@ -788,9 +809,10 @@ sub _set_error_message_locale { my ( $self, $params ) = @_; my @error_response = (); - my $locale = $self->_get_locale( $params ); + my $locale = $self->_get_locale( $params ); + + if ( not defined $locale or $locale eq "" ) { - if (not defined $locale or $locale eq "") { # Don't translate message if locale is not defined $locale = "C"; } @@ -802,27 +824,29 @@ sub _set_error_message_locale { my $rpc_request = joi->object->props( jsonrpc => joi->string->required, - method => $zm_validator->jsonrpc_method()->required, - id => joi->type([qw(null number string)])); + method => $zm_validator->jsonrpc_method()->required, + id => joi->type( [qw(null number string)] ) +); + sub jsonrpc_validate { my ( $self, $jsonrpc_request ) = @_; - my @error_rpc = $rpc_request->validate($jsonrpc_request); - if ((ref($jsonrpc_request) eq 'HASH' && !exists $jsonrpc_request->{id}) || @error_rpc) { + my @error_rpc = $rpc_request->validate( $jsonrpc_request ); + if ( ( ref( $jsonrpc_request ) eq 'HASH' && !exists $jsonrpc_request->{id} ) || @error_rpc ) { $self->_set_error_message_locale; return { jsonrpc => '2.0', - id => undef, - error => { - code => '-32600', + id => undef, + error => { + code => '-32600', message => 'The JSON sent is not a valid request object.', - data => "@error_rpc" + data => "@error_rpc" } - } + }; } - my $method_schema = $json_schemas{$jsonrpc_request->{method}}; - if (blessed $method_schema) { + my $method_schema = $json_schemas{ $jsonrpc_request->{method} }; + if ( blessed $method_schema) { $method_schema = $method_schema->compile; } @@ -833,24 +857,24 @@ sub jsonrpc_validate { if ( exists $method_schema->{required} and not exists $jsonrpc_request->{params} ) { return { jsonrpc => '2.0', - id => $jsonrpc_request->{id}, - error => { - code => '-32602', + id => $jsonrpc_request->{id}, + error => { + code => '-32602', message => "Missing 'params' object", } }; } elsif ( exists $jsonrpc_request->{params} ) { - my @error_response = $self->validate_params($method_schema, $jsonrpc_request->{params}); + my @error_response = $self->validate_params( $method_schema, $jsonrpc_request->{params} ); if ( scalar @error_response ) { return { jsonrpc => '2.0', - id => $jsonrpc_request->{id}, - error => { - code => '-32602', - message => decode_utf8(__ 'Invalid method parameter(s).'), - data => \@error_response + id => $jsonrpc_request->{id}, + error => { + code => '-32602', + message => decode_utf8( __ 'Invalid method parameter(s).' ), + data => \@error_response } }; } @@ -865,40 +889,43 @@ sub validate_params { push @error_response, $self->_set_error_message_locale( $params ); - if (blessed $method_schema) { + if ( blessed $method_schema) { $method_schema = $method_schema->compile; } - my $jv = JSON::Validator::Schema::Draft7->new->coerce('booleans,numbers,strings')->data($method_schema); - $jv->formats(Zonemaster::Backend::Validator::formats( $self->{config} )); + my $jv = JSON::Validator::Schema::Draft7->new->coerce( 'booleans,numbers,strings' )->data( $method_schema ); + $jv->formats( Zonemaster::Backend::Validator::formats( $self->{config} ) ); my @json_validation_error = $jv->validate( $params ); # Customize error message from json validation foreach my $err ( @json_validation_error ) { my $message = $err->message; - my @details = @{$err->details}; + my @details = @{ $err->details }; # Handle 'required' errors globally so it does not get overwritten - if ($details[1] eq 'required') { + if ( $details[1] eq 'required' ) { $message = N__ 'Missing property'; - } else { + } + else { my @path = split '/', $err->path, -1; - shift @path; # first item is an empty string + shift @path; # first item is an empty string my $found = 1; - my $data = Mojo::JSON::Pointer->new($method_schema); - - foreach my $p (@path) { - if ( $data->contains("/properties/$p") ) { - $data = $data->get("/properties/$p") - } elsif ( $p =~ /^\d+$/ and $data->contains("/items") ) { - $data = $data->get("/items") - } else { + my $data = Mojo::JSON::Pointer->new( $method_schema ); + + foreach my $p ( @path ) { + if ( $data->contains( "/properties/$p" ) ) { + $data = $data->get( "/properties/$p" ); + } + elsif ( $p =~ /^\d+$/ and $data->contains( "/items" ) ) { + $data = $data->get( "/items" ); + } + else { $found = 0; last; } - $data = Mojo::JSON::Pointer->new($data); + $data = Mojo::JSON::Pointer->new( $data ); } - if ($found and exists $data->data->{'x-error-message'}) { + if ( $found and exists $data->data->{'x-error-message'} ) { $message = $data->data->{'x-error-message'}; } } @@ -908,7 +935,9 @@ sub validate_params { } # Translate messages - @error_response = map { { %$_, ( message => decode_utf8 __ $_->{message} ) } } @error_response; + @error_response = map { + { %$_, ( message => decode_utf8 __ $_->{message} ) } + } @error_response; return @error_response; } diff --git a/lib/Zonemaster/Backend/TLD_URL.pm b/lib/Zonemaster/Backend/TLD_URL.pm new file mode 100644 index 000000000..2f0b9b1b9 --- /dev/null +++ b/lib/Zonemaster/Backend/TLD_URL.pm @@ -0,0 +1,246 @@ +package Zonemaster::Backend::TLD_URL; + +=head1 Zonemaster::Backend::TLD_URL + +This Perl module is the backend for the RPCAPI method "get_tld_url" + +=cut + +use strict; +use warnings; +use 5.14.2; + +use HTTP::Tiny; +use JSON::PP qw(decode_json); +use Readonly; + +# Zonemaster Modules +use Zonemaster::Engine; +use Zonemaster::Engine::Recursor; +use Zonemaster::Backend; +use Zonemaster::Backend::Config; +use Zonemaster::Backend::Validator qw[ untaint_tld_block untaint_tld_url_no_path untaint_tld_url_with_path untaint_tld_url_string ]; +use Zonemaster::Backend::Errors; + +Readonly my $IANA_RDAP_URL_BASE => "https://rdap.iana.org/domain"; +Readonly my $DNS_NAME_BASE_TXT_RECORD => "_url._zonemaster"; +Readonly my $SOURCE_BACKEND_CONF_STR => "BACKEND CONF"; +Readonly my $SOURCE_TXT_RECORD_STR => "TXT RECORD"; +Readonly my $SOURCE_IANA_RDAP_STR => "IANA RDAP"; + +=head2 process ($self, $domain) + +Processes the domain name ($domain) for Zonemaster::Backend::RPCAPI::get_tld_url +and returns a complete hash reference to be returned by the RPCAPI. +See L +for a specification of the features implemented in this module. + +=cut + +sub process { + my ( $self, $domain ) = @_; + + my %result; + my $timeout = $self->{config}->TLD_URL_SETTINGS_lookup_timeout; + my $include_source = $self->{config}->TLD_URL_SETTINGS_include_source; + my $enable_tld_url = $self->{config}->TLD_URL_SETTINGS_enable_tld_url; + my %overrides = $self->{config}->TLD_URL_OVERRIDE; + my @labels = split( /\./, $domain ); + my $tld = $labels[$#labels]; # Empty if $domain is root '.' + + # Empty response if the function is not enabled + unless ( $enable_tld_url ) { + $result{tld} = $tld if defined $tld and $tld ne ''; + return \%result; + } + + # Empty response if the domain is the root zone + if ( $domain eq '.' ) { + return \%result; + } + + # Empty response if the domain is just a TLD + if ( scalar @labels == 1 ) { + $result{tld} = $tld; + return \%result; + } + + # Check any override from the configuration + my $href_override_result = url_from_override( $domain, $tld, $include_source, \%overrides ); + return $href_override_result if %$href_override_result; + + # Do a lookup of "_url._zonemaster.$tld" + my $href_txt_record_result = url_from_txt_record( $domain, $tld, $timeout, $include_source ); + return $href_txt_record_result if %$href_txt_record_result; + + # Do an IANA RDAP lookup + my $href_rdap_lookup_result = url_from_rdap ( $tld, $timeout, $include_source ); + return $href_rdap_lookup_result if %$href_rdap_lookup_result; + + $result{tld} = $tld; + return \%result; +} + +=head2 url_from_override ($dom, $tld, $include_source, $href_or) + +Used by subroutine "process" to do the "dirty work" to process +any overrides in the configuration. + +The following variables are mandatory in the call: + +=over 8 + +=item * The domain name to be processed ($dom) + +=item * The TLD extracted from the domain name ($tld) + +=item * Boolean value whether source of URL should be indicated in the result ($include_source) + +=item * Reference of a HASH of the override data (if any) from the configuration file ($href_or) + +=back + +A HASH reference ready to be sent by the RPCAPI is returned. Or empty then not to be sent. + +=cut + +sub url_from_override { + my ($dom, $tld, $include_source, $href_or) = @_; + my $url; + my %result; + + if ( exists $$href_or{$tld} ) { + + if ( untaint_tld_block( $$href_or{$tld} ) ) { + $result{tld} = $tld; + } else { + $url = $$href_or{$tld}; + $url = $url . '/' if untaint_tld_url_no_path( $url ); + $url =~ s/\Q[DOMAIN]\E/$dom/; # If any "[DOMAIN]" + $result{tld} = $tld; + $result{url} = $url; + $result{source} = $SOURCE_BACKEND_CONF_STR if $include_source; + } + } + return \%result; +} + +=head2 url_from_txt_record ($dom, $tld, $timeout, $include_source) + +Used by subroutine "process" to do the "dirty work" to process +the TXT lookup and the postprocessing of it. + +The following variables are mandatory in the call: + +=over 8 + +=item * The domain name to be processed ($dom) + +=item * The TLD extracted from the domain name ($tld) + +=item * The time limit for the lookup ($timeout) + +=item * Boolean value whether source of URL should be indicated in the result ($include_source) + +=back + +A HASH reference ready to be sent by the RPCAPI is returned. Or empty then not to be sent. + +=cut + +sub url_from_txt_record { + my ( $dom, $tld, $timeout, $include_source ) = @_; + my $url; + my %result; + my $name = $DNS_NAME_BASE_TXT_RECORD . '.' . $tld; + my $packet; + + eval { + local $SIG{ALRM} = sub { die "alarm\n" }; + alarm $timeout; + $packet = Zonemaster::Engine::Recursor->recurse( $name, 'TXT' ); + alarm 0; + }; + # Use $packet if defined + if ( $packet and $packet->rcode eq q{NOERROR} ) { + my @rrs = $packet->get_records_for_name( q{TXT}, $name ); + my @txt_rdata = map { $_->txtdata() } @rrs; + if ( scalar ( @txt_rdata ) == 1 ) { # Ignore all if more than one + my $data = $txt_rdata[0]; + if ( untaint_tld_block( $data ) ) { # "[BLOCK]" + $result{tld} = $tld; + } elsif ( untaint_tld_url_no_path( $data ) ) { # URL without path + $result{tld} = $tld; + $result{url} = $data . '/'; + $result{source} = $SOURCE_TXT_RECORD_STR if $include_source; + } elsif ( untaint_tld_url_string( $data ) ) { # URL with path and possible "[DOMAIN]" + $data =~ s/\Q[DOMAIN]\E/$dom/; # If any "[DOMAIN]" + $result{tld} = $tld; + $result{url} = $data; + $result{source} = $SOURCE_TXT_RECORD_STR if $include_source; + } + } + } + return \%result; +} + +=head2 url_from_rdap ($tld, $timeout, $include_source ) + +Used by subroutine "process" to do the "dirty work" to process +the IANA RDAP lookup and the postprocessing of it. + +The following variables are mandatory in the call: + +=over 8 + + +=item * The TLD extracted from the domain name ($tld) + +=item * The time limit for the lookup ($timeout) + +=item * Boolean value whether source of URL should be indicated in the result ($include_source) + +=back + +A HASH reference ready to be sent by the RPCAPI is returned. Or empty then not to be sent. + +=cut + +sub url_from_rdap { + my ( $tld, $timeout, $include_source ) = @_; + my $url = $IANA_RDAP_URL_BASE . '/' . $tld; + my $response; + my @links = (); + my $link = ''; + my %result; + + eval { + local $SIG{ALRM} = sub { die "alarm\n" }; + alarm $timeout; + $response = HTTP::Tiny->new->get($url); + alarm 0; + }; + if ($@) { + if ( $@ eq "alarm\n" ) { + handle_exception( "Timeout looking $url up" ); + } else { + handle_exception( "Unexpected error looking $url up: $@" ); + } + } + if ($response->{success}) { + my $data = decode_json($response->{content}); + @links = map { $_->{href} } grep { ($_->{rel} // '') eq 'related' } @{ $data->{links} // [] }; + }; + if (scalar @links > 0) { + $links[0] = $links[0] . '/' if untaint_tld_url_no_path( $links[0] ); + $link = $links[0] if untaint_tld_url_with_path( $links[0] ); + if ( $link ) { + $result{tld} = $tld; + $result{url} = $link; + $result{source} = $SOURCE_IANA_RDAP_STR if $include_source; + } + } + return \%result; +} + +1; diff --git a/lib/Zonemaster/Backend/TestAgent.pm b/lib/Zonemaster/Backend/TestAgent.pm index aaf182e47..b32580fc9 100644 --- a/lib/Zonemaster/Backend/TestAgent.pm +++ b/lib/Zonemaster/Backend/TestAgent.pm @@ -1,9 +1,9 @@ package Zonemaster::Backend::TestAgent; -our $VERSION = '1.1.0'; -use strict; -use warnings; use 5.14.2; +use warnings; + +our $VERSION = '1.1.0'; use DBI qw(:utils); use JSON::PP; @@ -42,6 +42,7 @@ sub new { my $dbclass = Zonemaster::Backend::DB->get_db_class( $dbtype ); $self->{_db} = $dbclass->from_config( $config ); + $self->{_db}->assert_compatible_schema; $self->{_profiles} = Zonemaster::Backend::Config->load_profiles( # $config->PUBLIC_PROFILES, diff --git a/lib/Zonemaster/Backend/Validator.pm b/lib/Zonemaster/Backend/Validator.pm index 1d7f4f36c..7bd8e78c8 100644 --- a/lib/Zonemaster/Backend/Validator.pm +++ b/lib/Zonemaster/Backend/Validator.pm @@ -19,6 +19,7 @@ use Zonemaster::LDNS; our @EXPORT_OK = qw( untaint_abs_path untaint_bool + untaint_json_bool untaint_engine_type untaint_ip_address untaint_ipv4_address @@ -34,17 +35,26 @@ our @EXPORT_OK = qw( untaint_profile_name untaint_strictly_positive_int untaint_strictly_positive_millis + untaint_tld_label + untaint_tld_block + untaint_tld_url_no_path + untaint_tld_url_with_path + untaint_tld_url_string check_domain check_ip check_profile check_language_tag ); + + + our %EXPORT_TAGS = ( untaint => [ qw( untaint_abs_path untaint_bool + untaint_json_bool untaint_engine_type untaint_ip_address untaint_ipv4_address @@ -60,6 +70,11 @@ our %EXPORT_TAGS = ( untaint_profile_name untaint_strictly_positive_int untaint_strictly_positive_millis + untaint_tld_label + untaint_tld_block + untaint_tld_url_no_path + untaint_tld_url_with_path + untaint_tld_url_string ) ], format => [ @@ -114,11 +129,24 @@ Readonly my $RELAXED_DOMAIN_NAME_RE => qr/^[.]$|^.{2,254}$/; Readonly my $TEST_ID_RE => qr/^[0-9a-f]{16}$/; Readonly my $USERNAME_RE => qr/^[a-z0-9-.@]{1,50}$/i; +# RE for URL for TLD +Readonly my $TLD_LABEL_RE => qr/^([a-z][a-z]+|xn--[a-z0-9-][a-z0-9-]+)$/; # ASCII or A-label +Readonly my $TLD_BLOCK_RE => qr/^\Q[BLOCK]\E$/; # Blocking policy +Readonly my $TLD_URL_NO_PATH_RE => qr/^(http|https):\/\/[a-z0-9][a-z0-9.-]*[a-z0-9]$/; # URL without path +Readonly my $TLD_URL_WITH_PATH_RE => qr/^(http|https):\/\/[a-z0-9][a-z0-9.-]*[a-z0-9]\/[a-zA-Z0-9\/=?%_.&-]*$/; # URL with path +# URL with path and possibly "[DOMAIN]" variable +Readonly my $TLD_URL_STRING_RE => qr/^(http|https):\/\/[a-z0-9][a-z0-9.-]*[a-z0-9]\/[a-zA-Z0-9\/=?%_.&-]*(\[DOMAIN\])?[a-zA-Z0-9\/=?%_.&-]*$/; + # Boolean Readonly my $BOOL_TRUE_RE => qr/^(true|yes)$/i; Readonly my $BOOL_FALSE_RE => qr/^(false|no)$/i; Readonly my $BOOL_RE => qr/^$BOOL_TRUE_RE|$BOOL_FALSE_RE$/i; +# Boolean, only JSON values +Readonly my $BOOL_JSON_TRUE_RE => qr/^true$/i; +Readonly my $BOOL_JSON_FALSE_RE => qr/^false$/i; +Readonly my $BOOL_JSON_RE => qr/^$BOOL_JSON_TRUE_RE|$BOOL_JSON_FALSE_RE$/i; + sub joi { return JSON::Validator::Joi->new; } @@ -387,6 +415,63 @@ sub untaint_abs_path { return _untaint_pred( $value, \&file_name_is_absolute ); } +=head2 untaint_tld_label + +Accepts a TLD label in ASCII or an IDN TLD label as A-label. + +=cut + +sub untaint_tld_label { + my ( $value ) = @_; + return _untaint_pat( $value, $TLD_LABEL_RE ); +} + +=head2 untaint_tld_block + +Accepts a string for blocking (policy). + +=cut + +sub untaint_tld_block { + my ( $value ) = @_; + return _untaint_pat( $value, $TLD_BLOCK_RE ); +} + +=head2 untaint_tld_url_no_path + +Returns true if the string contains a URL with no path. See +L. + +=cut + +sub untaint_tld_url_no_path { + my ( $value ) = @_; + return _untaint_pat( $value, $TLD_URL_NO_PATH_RE ); +} + +=head2 untaint_tld_url_with_path + +Accepts a URL for TLD. + +=cut + +sub untaint_tld_url_with_path { + my ( $value ) = @_; + return _untaint_pat( $value, $TLD_URL_WITH_PATH_RE ); +} + +=head2 untaint_tld_url_string + +Accepts a URL for TLD. + +=cut + +sub untaint_tld_url_string { + my ( $value ) = @_; + return _untaint_pat( $value, $TLD_URL_STRING_RE ); +} + + =head2 untaint_engine_type Accepts the strings C<"MySQL">, C<"PostgreSQL"> and C<"SQLite">, @@ -524,6 +609,15 @@ sub untaint_bool { return $ret; } +sub untaint_json_bool { + my ( $value ) = @_; + + my $ret; + $ret = 1 if defined _untaint_pat( $value, $BOOL_JSON_TRUE_RE ); + $ret = 0 if defined _untaint_pat( $value, $BOOL_JSON_FALSE_RE ); + return $ret; +} + sub _untaint_pat { my ( $value, @patterns ) = @_; diff --git a/script/zmb b/script/zmb index 2f5d9f638..0263bc140 100755 --- a/script/zmb +++ b/script/zmb @@ -658,6 +658,43 @@ sub cmd_batch_status { } +=head2 get_tld_url + zmb [GLOBAL OPTIONS] get_tld_url [OPTIONS] + + Options: + --domain DOMAIN_NAME + +"--domain" is mandatory. The domain name must be provided with ASCII only. Any +IDN labels must be as A-labels. + +The command provides a URL to the TLD that the DOMAIN_NAME belongs to, if +available and if policy permits. If DOMAIN_NAME is the root or a TLD then no +URL will be provided. +=cut + +sub cmd_get_tld_url { + my @opts = @_; + + my $opt_domain_name; + + GetOptionsFromArray( + \@opts, + 'domain-name|d=s' => \$opt_domain_name, + ) or pod2usage( 2 ); + + my %params; + $params{domain} = $opt_domain_name; + + return to_jsonrpc( + id => 1, + method => 'get_tld_url', + params => \%params, + ); +} + + + + sub show_commands { my %specials = ( man => 'Show the full manual page.', diff --git a/script/zonemaster_backend_rpcapi.psgi b/script/zonemaster_backend_rpcapi.psgi index 78813417f..902a7337c 100644 --- a/script/zonemaster_backend_rpcapi.psgi +++ b/script/zonemaster_backend_rpcapi.psgi @@ -20,7 +20,7 @@ use Try::Tiny; BEGIN { $ENV{PERL_JSON_BACKEND} = 'JSON::PP'; undef $ENV{LANGUAGE}; -}; +} use Zonemaster::Backend::RPCAPI; use Zonemaster::Backend::Config; @@ -30,18 +30,24 @@ local $| = 1; Log::Any::Adapter->set( '+Zonemaster::Backend::Log', - log_level => $ENV{ZM_BACKEND_RPCAPI_LOGLEVEL}, - json => $ENV{ZM_BACKEND_RPCAPI_LOGJSON}, - stderr => 1 + log_level => $ENV{ZM_BACKEND_RPCAPI_LOGLEVEL}, + json => $ENV{ZM_BACKEND_RPCAPI_LOGJSON}, + stderr => 1, + with_pid => !$ENV{ZM_BACKEND_RPCAPI_NO_LOGPID}, + with_timestamp => !$ENV{ZM_BACKEND_RPCAPI_NO_LOGTIMESTAMP}, ); $SIG{__WARN__} = sub { - $log->warning(map s/^\s+|\s+$//gr, map s/\n/ /gr, @_); + $log->warning( map s/^\s+|\s+$//gr, map s/\n/ /gr, @_ ); }; -my $config = Zonemaster::Backend::Config->load_config(); +my $config = Zonemaster::Backend::Config->load_config(); +my $profiles = Zonemaster::Backend::Config->load_profiles( # + $config->PUBLIC_PROFILES, + $config->PRIVATE_PROFILES, +); -Zonemaster::Backend::Metrics->setup($config->METRICS_statsd_host, $config->METRICS_statsd_port); +Zonemaster::Backend::Metrics->setup( $config->METRICS_statsd_host, $config->METRICS_statsd_port ); Zonemaster::Engine::init_engine(); builder { @@ -49,198 +55,228 @@ builder { my $app = shift; # Make sure we can connect to the database - $config->new_DB(); + my $dbh = $config->new_DB; + + # Make sure the database has the expected schema version + $dbh->assert_compatible_schema; return $app; }; }; -my $handler = Zonemaster::Backend::RPCAPI->new( { config => $config } ); +my $handler = Zonemaster::Backend::RPCAPI->new( + config => $config, + db => $config->new_DB, + profiles => $profiles, +); my $router = router { ############## FRONTEND #################### connect "version_info" => { handler => $handler, - action => "version_info" + action => "version_info" }; # Experimental connect "system_versions" => { handler => $handler, - action => "system_versions" + action => "system_versions" }; connect "profile_names" => { handler => $handler, - action => "profile_names" + action => "profile_names" }; # Experimental connect "conf_profiles" => { handler => $handler, - action => "conf_profiles" + action => "conf_profiles" }; connect "get_language_tags" => { handler => $handler, - action => "get_language_tags" + action => "get_language_tags" }; # Experimental connect "conf_languages" => { handler => $handler, - action => "conf_languages" + action => "conf_languages" }; connect "get_host_by_name" => { handler => $handler, - action => "get_host_by_name" + action => "get_host_by_name" }; # Experimental connect "lookup_address_records" => { handler => $handler, - action => "lookup_address_records" + action => "lookup_address_records" }; connect "get_data_from_parent_zone" => { handler => $handler, - action => "get_data_from_parent_zone" + action => "get_data_from_parent_zone" }; # Experimental connect "lookup_delegation_data" => { handler => $handler, - action => "lookup_delegation_data" + action => "lookup_delegation_data" }; connect "start_domain_test" => { handler => $handler, - action => "start_domain_test" + action => "start_domain_test" }; # Experimental connect "job_create" => { handler => $handler, - action => "job_create" + action => "job_create" }; connect "test_progress" => { handler => $handler, - action => "test_progress" + action => "test_progress" }; # Experimental connect "job_status" => { handler => $handler, - action => "job_status" + action => "job_status" }; connect "get_test_params" => { handler => $handler, - action => "get_test_params" + action => "get_test_params" }; # Experimental connect "job_params" => { handler => $handler, - action => "job_params" + action => "job_params" }; connect "get_test_results" => { handler => $handler, - action => "get_test_results" + action => "get_test_results" }; # Experimental connect "job_results" => { handler => $handler, - action => "job_results" + action => "job_results" }; connect "get_test_history" => { handler => $handler, - action => "get_test_history" + action => "get_test_history" }; # Experimental connect "domain_history" => { handler => $handler, - action => "domain_history" + action => "domain_history" }; connect "batch_status" => { handler => $handler, - action => "batch_status" + action => "batch_status" + }; + + connect "get_tld_url" => { + handler => $handler, + action => "get_tld_url" }; }; if ( $config->RPCAPI_enable_user_create or $config->RPCAPI_enable_add_api_user ) { - $log->info('Enabling add_api_user method'); - $router->connect("add_api_user", { - handler => $handler, - action => "add_api_user" - }); - $router->connect("user_create", { - handler => $handler, - action => "user_create" - }); + $log->info( 'Enabling add_api_user method' ); + $router->connect( + "add_api_user", + { + handler => $handler, + action => "add_api_user" + } + ); + $router->connect( + "user_create", + { + handler => $handler, + action => "user_create" + } + ); } if ( $config->RPCAPI_enable_batch_create or $config->RPCAPI_enable_add_batch_job ) { - $log->info('Enabling add_batch_job method'); - $router->connect("add_batch_job", { - handler => $handler, - action => "add_batch_job" - }); - $router->connect("batch_create", { - handler => $handler, - action => "batch_create" - }); + $log->info( 'Enabling add_batch_job method' ); + $router->connect( + "add_batch_job", + { + handler => $handler, + action => "add_batch_job" + } + ); + $router->connect( + "batch_create", + { + handler => $handler, + action => "batch_create" + } + ); } -my $dispatch = JSON::RPC::Dispatch->new( - router => $router, -); +my $dispatch = JSON::RPC::Dispatch->new( router => $router, ); my $rpcapi_app = sub { - my $env = shift; - my $req = Plack::Request->new($env); - my $res = {}; - my $content = {}; + my $env = shift; + my $req = Plack::Request->new( $env ); + my $res = {}; + my $content = {}; my $json_error = ''; try { my $json = $req->content; - $content = decode_json($json); - } catch { - $json_error = (split /at \//, $_)[0]; - }; - - if ($json_error eq '') { - my $errors = $handler->jsonrpc_validate($content); - if ($errors ne '') { - $res = Plack::Response->new(200); - $res->content_type('application/json'); - $res->body( encode_json($errors) ); - $res->finalize; - } else { + $content = decode_json( $json ); + } + catch { + $json_error = ( split /at \//, $_ )[0]; + }; + + if ( $json_error eq '' ) { + my $errors = $handler->jsonrpc_validate( $content ); + if ( $errors ne '' ) { + $res = Plack::Response->new( 200 ); + $res->content_type( 'application/json' ); + $res->body( encode_json( $errors ) ); + $res->finalize; + } + else { local $log->context->{rpc_method} = $content->{method}; - $res = $dispatch->handle_psgi($env, $env->{REMOTE_ADDR}); - my $status = Zonemaster::Backend::Metrics->code_to_status(decode_json(@{@$res[2]}[0])->{error}->{code}); - Zonemaster::Backend::Metrics::increment("zonemaster.rpcapi.requests.$content->{method}.$status"); + $res = $dispatch->handle_psgi( $env, $env->{REMOTE_ADDR} ); + my $status = Zonemaster::Backend::Metrics->code_to_status( decode_json( @{ @$res[2] }[0] )->{error}->{code} ); + Zonemaster::Backend::Metrics::increment( "zonemaster.rpcapi.requests.$content->{method}.$status" ); $res; } - } else { - $res = Plack::Response->new(200); - $res->content_type('application/json'); - $res->body( encode_json({ + } + else { + $res = Plack::Response->new( 200 ); + $res->content_type( 'application/json' ); + $res->body( + encode_json( + { jsonrpc => '2.0', - id => undef, - error => { - code => '-32700', + id => undef, + error => { + code => '-32700', message => 'Invalid JSON was received by the server.', - data => "$json_error" - }}) ); + data => "$json_error" + } + } + ) + ); $res->finalize; } diff --git a/script/zonemaster_backend_testagent b/script/zonemaster_backend_testagent index e21ac0f82..d5d8ca61e 100755 --- a/script/zonemaster_backend_testagent +++ b/script/zonemaster_backend_testagent @@ -17,7 +17,7 @@ use Pod::Usage; use Getopt::Long; use POSIX; use Time::HiRes qw[time sleep gettimeofday tv_interval]; -use sigtrap qw(die normal-signals); +use sigtrap qw(die normal-signals); ### ### Compile-time stuff. @@ -43,15 +43,19 @@ my $loglevel; my $logjson; my $opt_outfile; my $opt_help; +my $opt_logpid = 1; +my $opt_logtimestamp = 1; GetOptions( - 'help!' => \$opt_help, - 'pidfile=s' => \$pidfile, - 'user=s' => \$user, - 'group=s' => \$group, - 'logfile=s' => \$logfile, - 'loglevel=s' => \$loglevel, - 'logjson!' => \$logjson, - 'outfile=s' => \$opt_outfile, + 'help!' => \$opt_help, + 'pidfile=s' => \$pidfile, + 'user=s' => \$user, + 'group=s' => \$group, + 'logfile=s' => \$logfile, + 'loglevel=s' => \$loglevel, + 'logjson!' => \$logjson, + 'outfile=s' => \$opt_outfile, + 'logpid!' => \$opt_logpid, + 'logtimestamp!' => \$opt_logtimestamp, ) or pod2usage( "Try '$0 --help' for more information." ); pod2usage( -verbose => 1 ) if $opt_help; @@ -64,13 +68,15 @@ $loglevel = lc $loglevel; Log::Any::Adapter->set( '+Zonemaster::Backend::Log', - log_level => $loglevel, - json => $logjson, - file => $logfile, + log_level => $loglevel, + json => $logjson, + file => $logfile, + with_pid => $opt_logpid, + with_timestamp => $opt_logtimestamp, ); $SIG{__WARN__} = sub { - $log->warning(map s/^\s+|\s+$//gr, map s/\n/ /gr, @_); + $log->warning( map s/^\s+|\s+$//gr, map s/\n/ /gr, @_ ); }; ### @@ -93,28 +99,25 @@ sub main { my $agent = Zonemaster::Backend::TestAgent->new( { config => $self->config } ); while ( !$caught_sigterm ) { - my $cleanup_timer = [ gettimeofday ]; + my $cleanup_timer = [gettimeofday]; $self->pm->reap_finished_children(); # Reaps terminated child processes $self->pm->on_wait(); # Sends SIGKILL to overdue child processes - Zonemaster::Backend::Metrics::gauge("zonemaster.testagent.maximum_processes", $self->pm->max_procs); - Zonemaster::Backend::Metrics::gauge("zonemaster.testagent.running_processes", scalar($self->pm->running_procs)); + Zonemaster::Backend::Metrics::gauge( "zonemaster.testagent.maximum_processes", $self->pm->max_procs ); + Zonemaster::Backend::Metrics::gauge( "zonemaster.testagent.running_processes", scalar( $self->pm->running_procs ) ); - Zonemaster::Backend::Metrics::timing("zonemaster.testagent.cleanup_duration_seconds", tv_interval($cleanup_timer) * 1000); + Zonemaster::Backend::Metrics::timing( "zonemaster.testagent.cleanup_duration_seconds", tv_interval( $cleanup_timer ) * 1000 ); - my $fetch_test_timer = [ gettimeofday ]; + my $fetch_test_timer = [gettimeofday]; my ( $test_id, $batch_id ); eval { - $self->db->process_unfinished_tests( - $self->config->ZONEMASTER_lock_on_queue, - $self->config->ZONEMASTER_max_zonemaster_execution_time, - ); + $self->db->process_unfinished_tests( $self->config->ZONEMASTER_lock_on_queue, $self->config->ZONEMASTER_max_zonemaster_execution_time, ); ( $test_id, $batch_id ) = $self->db->get_test_request( $self->config->ZONEMASTER_lock_on_queue ); - Zonemaster::Backend::Metrics::timing("zonemaster.testagent.fetchtests_duration_seconds", tv_interval($fetch_test_timer) * 1000); + Zonemaster::Backend::Metrics::timing( "zonemaster.testagent.fetchtests_duration_seconds", tv_interval( $fetch_test_timer ) * 1000 ); }; if ( $@ ) { $log->error( $@ ); @@ -126,22 +129,22 @@ sub main { $log->infof( "Test found: %s", $test_id ); if ( $self->pm->start( $test_id ) == 0 ) { # Forks off child process $log->infof( "Test starting: %s", $test_id ); - Zonemaster::Backend::Metrics::increment("zonemaster.testagent.tests_started"); - my $start_time = [ gettimeofday ]; + Zonemaster::Backend::Metrics::increment( "zonemaster.testagent.tests_started" ); + my $start_time = [gettimeofday]; eval { $agent->run( $test_id, $show_progress ) }; if ( $@ ) { chomp $@; - Zonemaster::Backend::Metrics::increment("zonemaster.testagent.tests_died"); + Zonemaster::Backend::Metrics::increment( "zonemaster.testagent.tests_died" ); $log->errorf( "Test died: %s: %s", $test_id, $@ ); - $self->db->process_dead_test( $test_id ) + $self->db->process_dead_test( $test_id ); } else { - Zonemaster::Backend::Metrics::increment("zonemaster.testagent.tests_completed"); + Zonemaster::Backend::Metrics::increment( "zonemaster.testagent.tests_completed" ); $log->infof( "Test completed: %s", $test_id ); } - Zonemaster::Backend::Metrics::timing("zonemaster.testagent.tests_duration_seconds", tv_interval($start_time) * 1000); + Zonemaster::Backend::Metrics::timing( "zonemaster.testagent.tests_duration_seconds", tv_interval( $start_time ) * 1000 ); $agent->reset(); - $self->pm->finish; # Terminates child process + $self->pm->finish; # Terminates child process } } else { @@ -157,31 +160,31 @@ sub main { } sub preflight_checks { + # Make sure we can load the configuration file - $log->debug("Starting pre-flight check"); + $log->debug( "Starting pre-flight check" ); my $initial_config = Zonemaster::Backend::Config->load_config(); - Zonemaster::Backend::Metrics->setup($initial_config->METRICS_statsd_host, $initial_config->METRICS_statsd_port); + Zonemaster::Backend::Metrics->setup( $initial_config->METRICS_statsd_host, $initial_config->METRICS_statsd_port ); # Validate the Zonemaster-Engine profile - Zonemaster::Backend::TestAgent->new( { config => $initial_config } ); + Zonemaster::Backend::Config->load_profiles( # + $initial_config->PUBLIC_PROFILES, + $initial_config->PRIVATE_PROFILES, + ); # Connect to the database $initial_config->new_DB(); - $log->debug("Completed pre-flight check"); + $log->debug( "Completed pre-flight check" ); return $initial_config; } - - my $initial_config; # Make sure the environment is alright before forking (only on startup) if ( grep /^foreground$|^restart$|^start$/, @ARGV ) { - eval { - $initial_config = preflight_checks(); - }; + eval { $initial_config = preflight_checks(); }; if ( $@ ) { $log->critical( "Aborting startup: $@" ); print STDERR "Aborting startup: $@"; @@ -217,8 +220,8 @@ my $daemon = Daemon::Control->with_plugins( qw( +Zonemaster::Backend::Config::DC ); $daemon->init_config( $ENV{PERLBREW_ROOT} . '/etc/bashrc' ) if ( $ENV{PERLBREW_ROOT} ); -$daemon->user($user) if $user; -$daemon->group($group) if $group; +$daemon->user( $user ) if $user; +$daemon->group( $group ) if $group; exit $daemon->run; @@ -258,7 +261,7 @@ When FILE is -, the log is written to standard output. =item B<--loglevel=LEVEL> -The location of the log level to use. +The threshold for emitting log entries. The allowed values are specified at L. @@ -266,6 +269,14 @@ The allowed values are specified at L. Enable JSON logging when specified. +=item B<--[no-]logtimestamp> + +Controls the inclusion of timestamp log entries. Default: enabled. + +=item B<--[no-]logpid> + +Controls the inclusion of PID in log entries. Default: enabled. + =item B One of the following: diff --git a/share/backend_config.ini b/share/backend_config.ini index 535228989..a1e62f108 100755 --- a/share/backend_config.ini +++ b/share/backend_config.ini @@ -48,3 +48,14 @@ locale = da_DK en_US es_ES fi_FI fr_FR nb_NO sl_SI sv_SE # Uncoment the following option to enable the metrics feature #statsd_host = localhost #statsd_port = 8125 + +[TLD URL SETTINGS] +#enable_tld_url = true +#lookup_timeout = 3 +#include_source = true + +[TLD URL OVERRIDE] +#xa = [BLOCK] +#xn--4cab6c = [BLOCK] +#xb = http://nic.xb/domain +#example = https://whoisweb.noc.example/domain/[DOMAIN] diff --git a/share/patch/patch_db_schema_version_1.pl b/share/patch/patch_db_schema_version_1.pl new file mode 100644 index 000000000..37eb718bc --- /dev/null +++ b/share/patch/patch_db_schema_version_1.pl @@ -0,0 +1,41 @@ +use 5.14.2; +use strict; +use warnings; + +use Readonly; +use Zonemaster::Backend::Config; + +Readonly my $TARGET_SCHEMA_VERSION => 1; +Readonly my $EXPECTED_SCHEMA_VERSION => $TARGET_SCHEMA_VERSION - 1; + +my $config = Zonemaster::Backend::Config->load_config(); +say "Configured database engine: ", $config->DB_engine; + +my $db = $config->new_DB(); +my $detected_version = $db->get_schema_version(); +say "Target database schema version: ", $TARGET_SCHEMA_VERSION; +say "Expected pre-migration schema version: ", $EXPECTED_SCHEMA_VERSION; +say "Detected database schema version: ", $detected_version; + +if ( $detected_version eq $TARGET_SCHEMA_VERSION ) { + say "Schema already at target version."; + exit 0; +} +elsif ( $detected_version ne $EXPECTED_SCHEMA_VERSION ) { + say "Schema version requirement not met!"; + exit 2; +} + +say "Starting database migration"; + +$db->dbh->do( + "CREATE TABLE IF NOT EXISTS schema_version ( + id INTEGER PRIMARY KEY, + version INTEGER NOT NULL, + CHECK (id = 1), + CHECK (version >= 1) + )" +); +$db->dbh->do( "INSERT INTO schema_version (id, version) VALUES (1, ?)", {}, $TARGET_SCHEMA_VERSION ); + +say "Migration done"; diff --git a/share/sv.po b/share/sv.po index e409b0247..d08059e28 100644 --- a/share/sv.po +++ b/share/sv.po @@ -2,8 +2,8 @@ msgid "" msgstr "" "Project-Id-Version: 1.0.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2023-05-21 21:29+0000\n" -"PO-Revision-Date: 2023-05-21 21:29+0000\n" +"POT-Creation-Date: 2026-03-02 17:08+0000\n" +"PO-Revision-Date: 2026-03-02 17:09+0000\n" "Last-Translator: mats.dufberg@iis.se\n" "Language-Team: Zonemaster project\n" "Language: sv\n" @@ -57,25 +57,6 @@ msgstr "Keytag måste vara ett positivt heltal" msgid "Domain name required" msgstr "Domännamn är obligatoriskt" -msgid "The domain name is IDNA invalid" -msgstr "Domännamnet är ogiltigt enligt IDN-standarden" - -msgid "" -"The domain name contains non-ascii characters and IDNA support is not " -"installed" -msgstr "" -"Domännamnet innehåller icke-ASCII-tecken, men stöd för IDN är inte " -"installerat" - -msgid "The domain name character(s) are not supported" -msgstr "Domännamnstecken stöds inte" - -msgid "The domain name contains consecutive dots" -msgstr "Domännamnet innehåller flera punkter i följd" - -msgid "The domain name or label is too long" -msgstr "Domännamnet eller en domännamnsdel är för långt" - msgid "Invalid language tag format" msgstr "Ogiltigt format på språkkoden" diff --git a/share/zm-rpcapi.service b/share/zm-rpcapi.service index 8765ddebe..bbd8252ed 100644 --- a/share/zm-rpcapi.service +++ b/share/zm-rpcapi.service @@ -4,10 +4,16 @@ After=network.target mariadb.service postgresql.service Wants=mariadb.service postgresql.service [Service] -Type=simple -ExecStart=/usr/local/bin/starman --listen=127.0.0.1:5000 --preload-app --user=zonemaster --group=zonemaster --pid=/run/zonemaster/zm-rpcapi.pid --error-log=/var/log/zonemaster/zm-rpcapi.log --daemonize /usr/local/bin/zonemaster_backend_rpcapi.psgi +Type=exec +User=zonemaster +Group=zonemaster +Environment=ZM_BACKEND_RPCAPI_NO_LOGPID=1 +Environment=ZM_BACKEND_RPCAPI_NO_LOGTIMESTAMP=1 KillSignal=SIGQUIT -PIDFile=/run/zonemaster/zm-rpcapi.pid +ExecStart=/usr/local/bin/starman \ + --listen=127.0.0.1:5000 \ + --preload-app \ + /usr/local/bin/zonemaster_backend_rpcapi.psgi [Install] WantedBy=multi-user.target diff --git a/share/zm-testagent.service b/share/zm-testagent.service index 6117875fe..272f7a767 100644 --- a/share/zm-testagent.service +++ b/share/zm-testagent.service @@ -4,10 +4,14 @@ After=network.target mariadb.service postgresql.service Wants=mariadb.service postgresql.service [Service] -Type=simple -ExecStart=/usr/local/bin/zonemaster_backend_testagent --logfile=/var/log/zonemaster/zm-testagent.log --outfile=/var/log/zonemaster/zm-testagent.out --pidfile=/run/zonemaster/zm-testagent.pid --user=zonemaster --group=zonemaster start -ExecStop=/usr/local/bin/zonemaster_backend_testagent --logfile=/var/log/zonemaster/zm-testagent.log --outfile=/var/log/zonemaster/zm-testagent.out --pidfile=/run/zonemaster/zm-testagent.pid --user=zonemaster --group=zonemaster stop -PIDFile=/run/zonemaster/zm-testagent.pid +Type=exec +User=zonemaster +Group=zonemaster +ExecStart=/usr/local/bin/zonemaster_backend_testagent \ + --logfile=- \ + --no-logpid \ + --no-logtimestamp \ + foreground [Install] WantedBy=multi-user.target diff --git a/t/TestUtil.pm b/t/TestUtil.pm index fd62de463..7366dd8aa 100644 --- a/t/TestUtil.pm +++ b/t/TestUtil.pm @@ -5,6 +5,7 @@ use warnings; use Test::More; +use Carp qw( croak ); use Zonemaster::Engine; use Zonemaster::Backend::Config; @@ -74,7 +75,8 @@ sub restore_datafile { die q{Stored data file missing} if not -r $datafile; Zonemaster::Engine->preload_cache( $datafile ); Zonemaster::Engine->profile->set( q{no_network}, 1 ); - } else { + } + else { diag "recording"; } } @@ -106,28 +108,33 @@ sub init_db { } sub create_rpcapi { - my ( $config ) = @_; + my ( $config, %opts ) = @_; - my $rpcapi; - eval { - $rpcapi = Zonemaster::Backend::RPCAPI->new( - { - dbtype => $db_backend, - config => $config, - } - ); - }; - if ( $@ ) { - diag explain( $@ ); - BAIL_OUT( 'Could not connect to database' ); + my $dbtype_opt = delete $opts{override_dbtype}; + if ( %opts ) { + croak 'Unrecognized options: ' . join( ', ', sort keys %opts ); } - if ( not $rpcapi->isa('Zonemaster::Backend::RPCAPI' ) ) { + my $profiles = Zonemaster::Backend::Config->load_profiles( # + $config->PUBLIC_PROFILES, + $config->PRIVATE_PROFILES, + ); + + my $dbtype = $dbtype_opt // $db_backend; + my $db = $config->new_DB( override_dbtype => $dbtype ); + + prepare_db( $db ); + + my $rpcapi = Zonemaster::Backend::RPCAPI->new( + config => $config, + db => $db, + profiles => $profiles, + ); + + if ( not $rpcapi->isa( 'Zonemaster::Backend::RPCAPI' ) ) { BAIL_OUT( 'Not a Zonemaster::Backend::RPCAPI object' ); } - prepare_db( $rpcapi->{db} ); - return $rpcapi; } @@ -178,11 +185,16 @@ file. Database tables are dropped and created anew. -=item create_rpcapi($config) +=item create_rpcapi($config, %opts) Returns a new Zonemaster::Backend::RPCAPI object using the provided C<$config> file. +The C option overrides the effective value of C. + +The value of C in C<$config> is ignored. The effective value is taken from the +C option, C environment variable, or C by default. + Database tables are dropped and created anew. =item create_testagent($config) diff --git a/t/config.t b/t/config.t index 2892253a2..81fff36df 100644 --- a/t/config.t +++ b/t/config.t @@ -57,6 +57,15 @@ subtest 'Everything but NoWarnings' => sub { number_of_processes_for_batch_testing = 40 lock_on_queue = 1 age_reuse_previous_test = 800 + + [TLD URL SETTINGS] + enable_tld_url = false + lookup_timeout = 6 + include_source = false + + [TLD URL OVERRIDE] + xa = [BLOCK] + xb = http://nic.xb/domain }; my $config = Zonemaster::Backend::Config->parse( $text ); isa_ok $config, 'Zonemaster::Backend::Config', 'parse() return value'; @@ -89,6 +98,14 @@ subtest 'Everything but NoWarnings' => sub { is $config->ZONEMASTER_number_of_processes_for_batch_testing, 40, 'set: ZONEMASTER.number_of_processes_for_batch_testing'; is $config->ZONEMASTER_lock_on_queue, 1, 'set: ZONEMASTER.lock_on_queue'; is $config->ZONEMASTER_age_reuse_previous_test, 800, 'set: ZONEMASTER.age_reuse_previous_test'; + + is $config->TLD_URL_SETTINGS_enable_tld_url, 0, 'TLD_URL_SETTINGS_enable_tld_url'; + is $config->TLD_URL_SETTINGS_lookup_timeout, 6, 'TLD_URL_SETTINGS_lookup_timeout'; + is $config->TLD_URL_SETTINGS_include_source, 0, 'TLD_URL_SETTINGS_include_source'; + eq_or_diff { $config->TLD_URL_OVERRIDE }, { # + xa => '[BLOCK]', + xb => 'http://nic.xb/domain' + }, }; subtest 'Default values' => sub { @@ -114,6 +131,12 @@ subtest 'Everything but NoWarnings' => sub { is $config->RPCAPI_enable_add_api_user, 0, 'default: RPCAPI.enable_add_api_user'; is $config->RPCAPI_enable_add_batch_job, 1, 'default: RPCAPI.enable_add_batch_job'; + + is $config->TLD_URL_SETTINGS_enable_tld_url, 1, 'TLD_URL_SETTINGS_enable_tld_url'; + is $config->TLD_URL_SETTINGS_lookup_timeout, 3, 'TLD_URL_SETTINGS_lookup_timeout'; + is $config->TLD_URL_SETTINGS_include_source, 1, 'TLD_URL_SETTINGS_include_source'; + eq_or_diff { $config->TLD_URL_OVERRIDE }, { # + }, }; SKIP: { @@ -810,6 +833,205 @@ subtest 'Everything but NoWarnings' => sub { } qr/PRIVATE PROFILES.*default/, 'die: Default profile in PRIVATE PROFILES'; + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL SETTINGS] + enable_tld_url = no + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Invalid value/, 'die: Invalid value of TLD_URL_SETTINGS_enable_tld_url'; + + + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL SETTINGS] + enable_tld_url = 5 + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Invalid value/, 'die: Invalid value of TLD_URL_SETTINGS_enable_tld_url'; + + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL SETTINGS] + lookup_timeout = -1 + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Invalid value/, 'die: Invalid value of TLD_URL_SETTINGS_lookup_timeout'; + + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL SETTINGS] + lookup_timeout = 0.5 + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Invalid value/, 'die: Invalid value of TLD_URL_SETTINGS_lookup_timeout'; + + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL SETTINGS] + include_source = yes + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Invalid value/, 'die: Invalid value of TLD_URL_SETTINGS_include_source'; + + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL SETTINGS] + include_source = 10 + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Invalid value/, 'die: Invalid value of TLD_URL_SETTINGS_include_source'; + + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL OVERRIDE] + xa = [BLOCK] + xa = http://nic.xb/domain + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Property not unique/, 'die: TLD_URL_OVERRIDE.xa not unique'; + + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL OVERRIDE] + xa = [BLOK] + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Invalid value/, 'die: Invalid value in URL string in [TLD URL OVERRIDE]'; + + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL OVERRIDE] + xa = http://nic.xb/domain/$dom + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Invalid value/, 'die: Invalid value in URL string in [TLD URL OVERRIDE]'; + + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL OVERRIDE] + xa-xb = [BLOCK] + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Invalid TLD label/, 'die: Invalid TLD string in [TLD URL OVERRIDE]'; + + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL OVERRIDE] + xa = http://nic.xb/domain1 + xa = http://nic.xb/domain2 + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Property not unique/, 'die: TLD_URL_OVERRIDE.xa not unique'; + + throws_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL OVERRIDE] + dev.xa = http://nic.xb/domain + }; + Zonemaster::Backend::Config->parse( $text ); + } + qr/Invalid TLD label/, 'die: Invalid TLD string in [TLD URL OVERRIDE]'; + + lives_ok { + my $text = q{ + [DB] + engine = SQLite + + [SQLITE] + database_file = /var/db/zonemaster.sqlite + + [TLD URL OVERRIDE] + xa = http://nic.xb + }; + Zonemaster::Backend::Config->parse( $text ); + } + 'URL needs no path [TLD URL OVERRIDE]'; + + subtest 'RPCAPI experimental aliases' => sub { subtest 'default values' => sub { my $text = q{ diff --git a/t/db_schema_version.t b/t/db_schema_version.t new file mode 100644 index 000000000..69a81455c --- /dev/null +++ b/t/db_schema_version.t @@ -0,0 +1,121 @@ +use strict; +use warnings; + +use Test::More; +use Test::Exception; +use Test::NoWarnings qw( had_no_warnings ); + +use File::ShareDir qw( dist_file ); +use File::Temp qw( tempdir ); + +my $t_path; + +BEGIN { + use File::Spec::Functions qw( rel2abs ); + use File::Basename qw( dirname ); + $t_path = dirname( rel2abs( $0 ) ); +} +use lib $t_path; + +use TestUtil; + +use Zonemaster::Backend::Config; +use Zonemaster::Backend::RPCAPI; +use Zonemaster::Backend::TestAgent; + +my $db_backend = TestUtil::db_backend(); + +my $tempdir = tempdir( CLEANUP => 1 ); +my $config = Zonemaster::Backend::Config->parse( <load_profiles( # + $config->PUBLIC_PROFILES, + $config->PRIVATE_PROFILES, +); + +my $dbclass = Zonemaster::Backend::DB->get_db_class( $db_backend ); +my $db = $dbclass->from_config( $config ); + +subtest 'newly created database' => sub { + $db->drop_tables(); + $db->create_schema(); + + my $schema_version = $db->get_schema_version; + like $schema_version, qr{^[1-9][0-9]*$}, 'should report schema version as an integer'; + + lives_ok { $db->assert_compatible_schema } 'should pass compatibility assertion'; + lives_ok { Zonemaster::Backend::TestAgent->new( { config => $config } ) } 'should be accepted by TestAgent constructor'; + lives_ok { Zonemaster::Backend::RPCAPI->new( config => $config, db => $config->new_DB, profiles => $profiles ) } 'should be accepted by RPCAPI constructor'; +}; + +subtest 'database with future schema version' => sub { + $db->drop_tables(); + $db->create_schema(); + $db->dbh->do( 'UPDATE schema_version SET version = ?', {}, $Zonemaster::Backend::DB::REQUIRED_SCHEMA_VERSION + 1 ); + + my $schema_version = $db->get_schema_version; + like $schema_version, qr{^[1-9][0-9]*$}, 'should report schema version as an integer'; + + dies_ok { $db->assert_compatible_schema } 'should fail compatibility assertion'; + dies_ok { Zonemaster::Backend::RPCAPI->new( config => $config, db => $config->new_DB, profiles => $profiles ) } 'should be rejected by RPCAPI constructor'; + dies_ok { Zonemaster::Backend::TestAgent->new( { config => $config } ) } 'should be rejected by TestAgent constructor'; +}; + +subtest 'database per Backend 11.2.0' => sub { + $db->drop_tables(); + $db->create_schema(); + $db->dbh->do( 'DROP TABLE schema_version' ); + + my $schema_version = $db->get_schema_version; + is $schema_version, 0, 'should be inferred as schema version 0'; + + dies_ok { $db->assert_compatible_schema } 'should fail compatibility assertion'; + dies_ok { Zonemaster::Backend::RPCAPI->new( config => $config, db => $config->new_DB, profiles => $profiles ) } 'should be rejected by RPCAPI constructor'; + dies_ok { Zonemaster::Backend::TestAgent->new( { config => $config } ) } 'should be rejected by TestAgent constructor'; +}; + +subtest 'database with unrecognized schema version table structure' => sub { + $db->drop_tables(); + $db->create_schema(); + $db->dbh->do( "DROP TABLE schema_version" ); + $db->dbh->do( "CREATE TABLE schema_version (foobar INTEGER)" ); + + dies_ok { $db->get_schema_version; } 'should die instead of reporting a schema version'; + dies_ok { $db->assert_compatible_schema } 'should fail compatibility assertion'; + dies_ok { Zonemaster::Backend::RPCAPI->new( config => $config, db => $config->new_DB, profiles => $profiles ) } 'should be rejected by RPCAPI constructor'; + dies_ok { Zonemaster::Backend::TestAgent->new( { config => $config } ) } 'should be rejected by TestAgent constructor'; +}; + +subtest 'database with empty schema version table' => sub { + $db->drop_tables(); + $db->create_schema(); + $db->dbh->do( "DELETE FROM schema_version" ); + + dies_ok { $db->get_schema_version; } 'should die instead of reporting a schema version'; + dies_ok { $db->assert_compatible_schema } 'should fail compatibility assertion'; + dies_ok { Zonemaster::Backend::RPCAPI->new( config => $config, db => $config->new_DB, profiles => $profiles ) } 'should be rejected by RPCAPI constructor'; + dies_ok { Zonemaster::Backend::TestAgent->new( { config => $config } ) } 'should be rejected by TestAgent constructor'; +}; + +had_no_warnings; +done_testing; diff --git a/t/idn.data b/t/idn.data index 3f7bcc13c..7094bac44 100644 Binary files a/t/idn.data and b/t/idn.data differ diff --git a/t/idn.t b/t/idn.t index bcda6563b..aeb26ac3c 100644 --- a/t/idn.t +++ b/t/idn.t @@ -22,7 +22,7 @@ use Zonemaster::Backend::Config; my $db_backend = TestUtil::db_backend(); my $datafile = "$t_path/idn.data"; -TestUtil::restore_datafile( $datafile ); +TestUtil::restore_datafile( $datafile ) unless ( $ENV{ZONEMASTER_RECORD} ); my $tempdir = tempdir( CLEANUP => 1 ); @@ -121,6 +121,6 @@ subtest 'test IDN nameserver' => sub { }; }; -TestUtil::save_datafile( $datafile ); +TestUtil::save_datafile( $datafile ) if ( $ENV{ZONEMASTER_RECORD} ); done_testing(); diff --git a/t/log.t b/t/log.t new file mode 100644 index 000000000..d511aa54d --- /dev/null +++ b/t/log.t @@ -0,0 +1,164 @@ +#!perl +use strict; +use warnings FATAL => 'all'; +use Test::More; + +use Capture::Tiny qw( capture ); +use File::Slurp qw( slurp ); +use File::Temp qw( tempdir ); +use JSON::PP qw( decode_json ); +use Log::Any::Adapter; +use Test::Fatal qw( exception ); +use Zonemaster::Backend::Log; + +subtest 'render entry in text format' => sub { + my $stdout = capture { + my $logger = Zonemaster::Backend::Log->new; + $logger->structured( 'error', 'unit.test', 'text message', { request_id => 'abc123' }, ); + }; + + like $stdout, qr{ + \A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z # timestamp + \s+\[\d+\] # pid + \s+\[ERROR\] # log level + \s+\[unit[.]test\] # category + \s+text[ ]message # message + }x, 'text log entry contains timestamp, pid, level, category and message',; + + like $stdout, qr/Extra parameters:/, 'extra parameters are appended'; + like $stdout, qr/request_id/, 'extra parameter key is present'; + like $stdout, qr/abc123/, 'extra parameter value is present'; +}; + +subtest 'render entry in text format without pid' => sub { + my $stdout = capture { + my $logger = Zonemaster::Backend::Log->new( with_pid => 0 ); + $logger->structured( 'error', 'unit.test', 'text message', { request_id => 'abc123' }, ); + }; + + like $stdout, qr{ + \A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z # timestamp + # no pid + \s+\[ERROR\] # log level + \s+\[unit[.]test\] # category + \s+text[ ]message # message + }x, 'text log entry contains timestamp, level, category and message, but no pid',; +}; + +subtest 'render entry in text format without timestamp' => sub { + my $stdout = capture { + my $logger = Zonemaster::Backend::Log->new( with_timestamp => 0 ); + $logger->structured( 'error', 'unit.test', 'text message', { request_id => 'abc123' }, ); + }; + + like $stdout, qr{ + # no timestamp + \A\[\d+\] # pid + \s+\[ERROR\] # log level + \s+\[unit[.]test\] # category + \s+text[ ]message # message + }x, 'text log entry contains pid, level, category and message, but no timestamp',; +}; + +subtest 'render entry in JSON format' => sub { + my $stdout = capture { + my $logger = Zonemaster::Backend::Log->new( json => 1 ); + $logger->structured( 'error', 'unit.test', 'json message', { request_id => 'def456' }, ); + }; + + my $entry = decode_json( $stdout ); + + is $entry->{level}, 'error', 'level is serialized'; + is $entry->{category}, 'unit.test', 'category is serialized'; + is $entry->{message}, 'json message', 'message is serialized'; + is $entry->{request_id}, 'def456', 'structured data is serialized'; + + like $entry->{timestamp}, qr/\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z\z/, 'timestamp is serialized as UTC ISO-like string'; + + is( $entry->{pid}, $$, 'pid is serialized' ); +}; + +subtest 'rejects unknown textual log level' => sub { + my $error = exception { + Zonemaster::Backend::Log->new( log_level => 'not-a-level' ); + }; + + like $error, qr/Unrecognized log level not-a-level/, 'unknown textual log level dies'; +}; + +subtest 'redirect output to stderr' => sub { + my ( $stdout, $stderr ) = capture { + my $logger = Zonemaster::Backend::Log->new( stderr => 1 ); + $logger->structured( 'error', 'unit.test', 'message' ); + }; + + is $stdout, '', 'nothing was written to stdout'; + like $stderr, qr/\[ERROR\]/, 'entry was written to stderr'; +}; + +subtest 'redirect output to file' => sub { + my $dir = tempdir( CLEANUP => 1 ); + my $file = "$dir/backend.log"; + my ( $stdout, $stderr ) = capture { + my $logger = Zonemaster::Backend::Log->new( file => $file ); + $logger->structured( 'error', 'unit.test', 'message' ); + }; + my $content = slurp( $file ); + + is $stdout, '', 'nothing was written to stdout'; + is $stderr, '', 'nothing was written to stderr'; + like $content, qr/\[ERROR\]/, 'entry was written to file'; +}; + +subtest 'works as a Log::Any adapter' => sub { + Log::Any::Adapter->set( + { lexically => \my $adapter_scope }, + '+Zonemaster::Backend::Log', + log_level => 'debug', + json => 1, + ); + + my $log = Log::Any->get_logger( category => 'zonemaster.backend.log.test' ); + + ok !$log->is_trace, 'trace is disabled through Log::Any'; + ok $log->is_debug, 'debug is enabled through Log::Any'; + ok $log->is_info, 'info is enabled through Log::Any'; + + my $stdout = capture { + $log->info( 'message through Log::Any', { request_id => 'abc123' }, ); + }; + + my $entry; + is + exception { $entry = decode_json( $stdout ) }, + undef, + 'a single valid JSON value was written'; + + is $entry->{level}, 'info', 'Log::Any level reached backend logger'; + is $entry->{category}, 'zonemaster.backend.log.test', 'Log::Any category reached backend logger'; + is $entry->{message}, 'message through Log::Any', 'message reached backend logger'; + is $entry->{request_id}, 'abc123', 'structured data reached backend logger'; + is $entry->{pid}, $$, 'pid was added by backend logger'; + ok $entry->{timestamp}, 'timestamp was added by backend logger'; +}; + +subtest 'Log::Any level detection and filtering use backend log_level' => sub { + Log::Any::Adapter->set( { lexically => \my $adapter_scope }, '+Zonemaster::Backend::Log', log_level => 'warning', ); + + my $log = Log::Any->get_logger( category => 'zonemaster.backend.log.test' ); + + ok !$log->is_info, 'info is disabled through Log::Any'; + ok $log->is_error, 'error is enabled through Log::Any'; + + my $stdout = capture { + $log->info( 'filtered message' ); + $log->warn( 'visible message 1' ); + $log->error( 'visible message 2' ); + }; + + like $stdout, qr/\[ERROR\]/, 'error entry was written'; + like $stdout, qr/\[WARNING\]/, 'warn entry was written'; + unlike $stdout, qr/\[INFO\]/, 'info entry was skipped'; +}; + +done_testing; diff --git a/t/parameters_validation.t b/t/parameters_validation.t index f0a5331eb..097acf55e 100644 --- a/t/parameters_validation.t +++ b/t/parameters_validation.t @@ -1,11 +1,20 @@ -use strict; -use warnings; use 5.14.2; +use warnings; use utf8; use Test::More tests => 4; use Test::NoWarnings; +my $t_path; + +BEGIN { + use File::Spec::Functions qw( rel2abs ); + use File::Basename qw( dirname ); + $t_path = dirname( rel2abs( $0 ) ); +} +use lib $t_path; +use TestUtil; + use Cwd; use File::Temp qw[tempdir]; use Zonemaster::Backend::Config; @@ -27,12 +36,7 @@ database_file = $tempdir/zonemaster.sqlite test = $cwd/t/test_profile.json EOF -my $rpcapi = Zonemaster::Backend::RPCAPI->new( - { - dbtype => $config->DB_engine, - config => $config, - } -); +my $rpcapi = TestUtil::create_rpcapi( $config, override_dbtype => $config->DB_engine ); sub test_validation { my ( $method_name, $method_schema, $test_cases ) = @_; diff --git a/t/rpc_validation.t b/t/rpc_validation.t index de8a4f9e4..bb6c6b352 100644 --- a/t/rpc_validation.t +++ b/t/rpc_validation.t @@ -1,11 +1,20 @@ -use strict; -use warnings; use 5.14.2; +use warnings; use utf8; use Test::More tests => 30; use Test::NoWarnings; +my $t_path; + +BEGIN { + use File::Spec::Functions qw( rel2abs ); + use File::Basename qw( dirname ); + $t_path = dirname( rel2abs( $0 ) ); +} +use lib $t_path; +use TestUtil; + use Cwd; use File::Temp qw[tempdir]; use Zonemaster::Backend::Config; @@ -31,12 +40,7 @@ database_file = $tempdir/zonemaster.sqlite test = $cwd/t/test_profile.json EOF -my $rpcapi = Zonemaster::Backend::RPCAPI->new( - { - dbtype => $config->DB_engine, - config => $config, - } -); +my $rpcapi = TestUtil::create_rpcapi( $config, override_dbtype => $config->DB_engine ); ### ### JSONRPC request object construction helper diff --git a/t/test01.data b/t/test01.data index 5f0d1f801..3789a2816 100644 Binary files a/t/test01.data and b/t/test01.data differ diff --git a/t/test01.t b/t/test01.t index 9fe41c1bf..f3fdf0191 100644 --- a/t/test01.t +++ b/t/test01.t @@ -22,7 +22,7 @@ use Zonemaster::Backend::Config; my $db_backend = TestUtil::db_backend(); my $datafile = "$t_path/test01.data"; -TestUtil::restore_datafile( $datafile ); +TestUtil::restore_datafile( $datafile ) unless ( $ENV{ZONEMASTER_RECORD} ); my $tempdir = tempdir( CLEANUP => 1 ); @@ -208,9 +208,9 @@ subtest 'API calls' => sub { my $ds_value = { 'algorithm' => 13, - 'digest' => '1303e8da8fb60db500d5bea1ee5dc9a2bcc93dfe2fc43d346576658feccf5749', # must match case + 'digest' => '8163abf45792942cf4ee38cca31f6a6832fcdc6d402338fc687827690c4132f6', # must match case 'digtype' => 2, - 'keytag' => 29133 + 'keytag' => 65381 }; is( scalar( @{ $res->{ds_list} } ), 1, 'Has only one DS set' ); is_deeply( $res->{ds_list}[0], $ds_value, 'Has correct DS values' ); @@ -420,6 +420,6 @@ subtest 'normalize "domain" column' => sub { } }; -TestUtil::save_datafile( $datafile ); +TestUtil::save_datafile( $datafile ) if ( $ENV{ZONEMASTER_RECORD} ); done_testing(); diff --git a/t/test_validate_syntax.t b/t/test_validate_syntax.t index 84faea46a..411a310c0 100644 --- a/t/test_validate_syntax.t +++ b/t/test_validate_syntax.t @@ -1,11 +1,20 @@ -use strict; -use warnings; use 5.14.2; +use warnings; use utf8; use Test::More tests => 2; use Test::NoWarnings; +my $t_path; + +BEGIN { + use File::Spec::Functions qw( rel2abs ); + use File::Basename qw( dirname ); + $t_path = dirname( rel2abs( $0 ) ); +} +use lib $t_path; +use TestUtil; + use Encode; use File::ShareDir qw[dist_file]; use JSON::PP; @@ -26,12 +35,7 @@ database_file = $tempdir/zonemaster.sqlite locale = en_US fr_FR da_DK fi_FI nb_NO sl_SI sv_SE EOF -my $engine = Zonemaster::Backend::RPCAPI->new( - { - dbtype => $config->DB_engine, - config => $config, - } -); +my $engine = TestUtil::create_rpcapi( $config, override_dbtype => $config->DB_engine ); sub start_domain_validate_params { return $engine->validate_params( $Zonemaster::Backend::RPCAPI::json_schemas{start_domain_test}, @_ ); diff --git a/t/validator.t b/t/validator.t index 917e7e758..6004a3244 100644 --- a/t/validator.t +++ b/t/validator.t @@ -212,4 +212,99 @@ subtest 'Everything but NoWarnings' => sub { is scalar untaint_strictly_positive_millis( '-1' ), undef, 'reject: -1'; ok !tainted( untaint_strictly_positive_millis( taint( '0.5' ) ) ), 'launder taint'; }; + + subtest 'untaint_tld_label' => sub { + is scalar untaint_tld_label( 'ax' ), 'ax', 'accept: ax'; + is scalar untaint_tld_label( 'globaltld' ), 'globaltld', 'accept: globaltld'; + is scalar untaint_tld_label( 'xn--rksmrgs-5wao1o' ), 'xn--rksmrgs-5wao1o', 'accept: xn--rksmrgs-5wao1o'; + is scalar untaint_tld_label( 'AX' ), undef, 'reject: AX'; + is scalar untaint_tld_label( 'Global' ), undef, 'reject: Global'; + is scalar untaint_tld_label( 'Räksmörgås' ), undef, 'reject: Räksmörgås'; + is scalar untaint_tld_label( 'non-tld' ), undef, 'reject: non-tld'; + is scalar untaint_tld_label( 'xx--rksmrgs-5wao1o' ), undef, 'reject: xx--rksmrgs-5wao1o'; + is scalar untaint_tld_label( 'xn--' ), undef, 'reject: xn--'; + is scalar untaint_tld_label( 'test.xa' ), undef, 'reject: test.xa'; + is scalar untaint_tld_label( 'x' ), undef, 'reject: a'; + is scalar untaint_tld_label( 'ax.' ), undef, 'reject: "ax."'; + is scalar untaint_tld_label( '.' ), undef, 'reject: "."'; + is scalar untaint_tld_label( '' ), undef, 'reject: ""'; + ok !tainted( untaint_tld_label( taint( 'ax' ) ) ), 'launder taint'; + }; + + subtest 'untaint_tld_block' => sub { + is scalar untaint_tld_block( '[BLOCK]' ), '[BLOCK]', 'accept: [BLOCK]'; + is scalar untaint_tld_block( '[BLOK]' ), undef, 'reject: [BLOK]'; + is scalar untaint_tld_block( '[ BLOCK]' ), undef, 'reject: [ BLOCK]'; + is scalar untaint_tld_block( '[BLOCK ]' ), undef, 'reject: [BLOCK ]'; + is scalar untaint_tld_block( '[block]' ), undef, 'reject: [block]'; + ok !tainted( untaint_tld_block( taint( '[BLOCK]' ) ) ), 'launder taint'; + }; + + subtest 'untaint_tld_url_no_path' => sub { + is scalar untaint_tld_url_no_path( 'https://domain.nic.xa' ), 'https://domain.nic.xa', 'accept: https://domain.nic.xa'; + is scalar untaint_tld_url_no_path( 'http://domain.nic.xa' ), 'http://domain.nic.xa', 'accept: http://domain.nic.xa'; + is scalar untaint_tld_url_no_path( 'https://xn--rksmrgs-5wao1o.se' ), 'https://xn--rksmrgs-5wao1o.se', 'accept: https://xn--rksmrgs-5wao1o.se'; + is scalar untaint_tld_url_no_path( 'https:/domain.nic.xa' ), undef, 'reject: https:/domain.nic.xa'; + is scalar untaint_tld_url_no_path( 'ftp://domain.nic.xa' ), undef, 'reject: ftp://domain.nic.xa'; + is scalar untaint_tld_url_no_path( 'https://domain.nic.xa/' ), undef, 'reject: https://domain.nic.xa/'; + is scalar untaint_tld_url_no_path( 'https://domain.nic.xa/domain' ), undef, 'reject: https://domain.nic.xa/domain'; + is scalar untaint_tld_url_no_path( 'https://DOMAIN_NIC.XA' ), undef, 'reject: https://DOMAIN_NIC.XA'; + ok !tainted( untaint_tld_url_no_path( taint( 'https://domain.nic.xa' ) ) ), 'launder taint'; + }; + + subtest 'untaint_tld_url_with_path' => sub { + is scalar untaint_tld_url_with_path( 'https://domain.nic.xa/' ), 'https://domain.nic.xa/', 'accept: https://domain.nic.xa/'; + is scalar untaint_tld_url_with_path( 'http://domain.nic.xa/' ), 'http://domain.nic.xa/', 'accept: http://domain.nic.xa/'; + is scalar untaint_tld_url_with_path( 'https://xn--rksmrgs-5wao1o.se/' ), 'https://xn--rksmrgs-5wao1o.se/', 'accept: https://xn--rksmrgs-5wao1o.se/'; + is scalar untaint_tld_url_with_path( 'https://domain.nic.xa/search/domain' ), 'https://domain.nic.xa/search/domain', 'accept: https://domain.nic.xa/search/domain'; + is scalar untaint_tld_url_with_path( 'https://domain.nic.xa/SEARCH/domain' ), 'https://domain.nic.xa/SEARCH/domain', 'accept: https://domain.nic.xa/SEARCH/domain'; + is scalar untaint_tld_url_with_path( 'https://domain.nic.xa/search=now/domain' ), 'https://domain.nic.xa/search=now/domain', 'accept: https://domain.nic.xa/search=now/domain'; + is scalar untaint_tld_url_with_path( 'https://domain.nic.xa/search=0123456789/domain' ), 'https://domain.nic.xa/search=0123456789/domain', 'accept: https://domain.nic.xa/search=0123456789/domain'; + is scalar untaint_tld_url_with_path( 'https://domain.nic.xa/search?now/domain' ), 'https://domain.nic.xa/search?now/domain', 'accept: https://domain.nic.xa/search?now/domain'; + is scalar untaint_tld_url_with_path( 'https://domain.nic.xa/search%now/domain' ), 'https://domain.nic.xa/search%now/domain', 'accept: https://domain.nic.xa/search%now/domain'; + is scalar untaint_tld_url_with_path( 'https://domain.nic.xa/search_now.now/domain' ), 'https://domain.nic.xa/search_now.now/domain', 'accept: https://domain.nic.xa/search_now.now/domain'; + is scalar untaint_tld_url_with_path( 'https://domain.nic.xa/search&now-now/domain' ), 'https://domain.nic.xa/search&now-now/domain', 'accept: https://domain.nic.xa/search&now-now/domain'; + is scalar untaint_tld_url_with_path( 'https://domain.nic.xa/search$now-now/domain' ), undef, 'reject: https://domain.nic.xa/search$now-now/domain'; + is scalar untaint_tld_url_with_path( 'https://DOMAIN.NIC.XA/' ), undef, 'reject: https://DOMAIN.NIC.XA/'; + is scalar untaint_tld_url_with_path( 'https://domain.nic.xa/[DOMAIN]' ), undef, 'reject: https://domain.nic.xa/[DOMAIN]'; + ok !tainted( untaint_tld_url_with_path( taint( 'https://domain.nic.xa/' ) ) ), 'launder taint'; + }; + + subtest 'untaint_tld_url_string' => sub { + is scalar untaint_tld_url_string( 'https://domain.nic.xa/[DOMAIN]' ), 'https://domain.nic.xa/[DOMAIN]', 'accept: https://domain.nic.xa/[DOMAIN]'; + is scalar untaint_tld_url_string( 'http://domain.nic.xa/[DOMAIN]' ), 'http://domain.nic.xa/[DOMAIN]', 'accept: http://domain.nic.xa/[DOMAIN]'; + is scalar untaint_tld_url_string( 'https://xn--rksmrgs-5wao1o.se/' ), 'https://xn--rksmrgs-5wao1o.se/', 'accept: https://xn--rksmrgs-5wao1o.se/'; + is scalar untaint_tld_url_string( 'https://domain.nic.xa/search/[DOMAIN]/domain' ), 'https://domain.nic.xa/search/[DOMAIN]/domain', 'accept: https://domain.nic.xa/search/[DOMAIN]/domain'; + is scalar untaint_tld_url_string( 'https://domain.nic.xa/SEARCH/domain' ), 'https://domain.nic.xa/SEARCH/domain', 'accept: https://domain.nic.xa/SEARCH/domain'; + is scalar untaint_tld_url_string( 'https://domain.nic.xa/search=now/domain' ), 'https://domain.nic.xa/search=now/domain', 'accept: https://domain.nic.xa/search=now/domain'; + is scalar untaint_tld_url_string( 'https://domain.nic.xa/search=0123456789/domain' ), 'https://domain.nic.xa/search=0123456789/domain', 'accept: https://domain.nic.xa/search=0123456789/domain'; + is scalar untaint_tld_url_string( 'https://domain.nic.xa/search?now/domain' ), 'https://domain.nic.xa/search?now/domain', 'accept: https://domain.nic.xa/search?now/domain'; + is scalar untaint_tld_url_string( 'https://domain.nic.xa/search%now/domain' ), 'https://domain.nic.xa/search%now/domain', 'accept: https://domain.nic.xa/search%now/domain'; + is scalar untaint_tld_url_string( 'https://domain.nic.xa/search_now.now/domain' ), 'https://domain.nic.xa/search_now.now/domain', 'accept: https://domain.nic.xa/search_now.now/domain'; + is scalar untaint_tld_url_string( 'https://domain.nic.xa/search&now-now/domain' ), 'https://domain.nic.xa/search&now-now/domain', 'accept: https://domain.nic.xa/search&now-now/domain'; + is scalar untaint_tld_url_string( 'https://domain.nic.xa/search$now-now/domain' ), undef, 'reject: https://domain.nic.xa/search&now-now/domain'; + is scalar untaint_tld_url_string( 'https://domain.nic.xa/[domain]' ), undef, 'reject: https://domain.nic.xa/[domain]'; + ok !tainted( untaint_tld_url_string( taint( 'https://domain.nic.xa/[DOMAIN]' ) ) ), 'launder taint'; + }; + + subtest 'untaint_json_bool' => sub { + is scalar untaint_json_bool( 'true' ), 1, 'accept: true'; + is scalar untaint_json_bool( 'false' ), 0, 'accept: false'; + is scalar untaint_json_bool( 'yes' ), undef, 'reject: yes'; + is scalar untaint_json_bool( 'no' ), undef, 'reject: no'; + is scalar untaint_json_bool( '1' ), undef, 'reject: 1'; + is scalar untaint_json_bool( '0' ), undef, 'reject: 0'; + ok !tainted( untaint_json_bool( taint( 'true' ) ) ), 'launder taint'; + }; + + subtest 'untaint_bool' => sub { + is scalar untaint_bool( 'true' ), 1, 'accept: true'; + is scalar untaint_bool( 'false' ), 0, 'accept: false'; + is scalar untaint_bool( 'yes' ), 1, 'accept: yes'; + is scalar untaint_bool( 'no' ), 0, 'accept: no'; + is scalar untaint_bool( '1' ), undef, 'reject: 1'; + is scalar untaint_bool( '0' ), undef, 'reject: 0'; + ok !tainted( untaint_bool( taint( 'true' ) ) ), 'launder taint'; + }; + }; diff --git a/zonemaster_launch b/zonemaster_launch deleted file mode 100755 index 0d62eb424..000000000 --- a/zonemaster_launch +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh - - -case $1 in - - cli) - shift 1 - zonemaster-cli $@ - ;; - - zmb) - shift 1 - zmb $@ - ;; - - zmtest) - shift 1 - zmtest $@ - ;; - - rpcapi) - /usr/local/bin/starman --listen=0.0.0.0:5000 --preload-app --user=zonemaster --group=zonemaster /usr/local/bin/zonemaster_backend_rpcapi.psgi - - ;; - - testagent) - /usr/local/bin/zonemaster_backend_testagent -user=zonemaster --group=zonemaster foreground - ;; - - full) - exec /init - ;; - *) - echo "'$1' is not a valid option. - Available options: - - cli : pass argument to zonemaster-cli then quit - - full : start both rpcapi & testagent - - rpcapi - - testagent - - zmb - - zmtest - " - ;; -esac;