diff --git a/ament_cmake_yamllint/CHANGELOG.rst b/ament_cmake_yamllint/CHANGELOG.rst new file mode 100644 index 000000000..e69de29bb diff --git a/ament_cmake_yamllint/CMakeLists.txt b/ament_cmake_yamllint/CMakeLists.txt new file mode 100644 index 000000000..504693b7f --- /dev/null +++ b/ament_cmake_yamllint/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.5) + +project(ament_cmake_yamllint NONE) + +find_package(ament_cmake_core REQUIRED) +find_package(ament_cmake_test REQUIRED) + +ament_package() + +install( + DIRECTORY cmake + DESTINATION share/${PROJECT_NAME} +) + +if(BUILD_TESTING) + find_package(ament_cmake_copyright REQUIRED) + ament_copyright() + + find_package(ament_cmake_lint_cmake REQUIRED) + ament_lint_cmake() + + find_package(ament_cmake_xmllint REQUIRED) + ament_xmllint() +endif() diff --git a/ament_cmake_yamllint/cmake/ament_yamllint.cmake b/ament_cmake_yamllint/cmake/ament_yamllint.cmake new file mode 100644 index 000000000..871c528d8 --- /dev/null +++ b/ament_cmake_yamllint/cmake/ament_yamllint.cmake @@ -0,0 +1,69 @@ +# Copyright 2014-2018 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Add a test to check YAML files with yamllint. +# +# :param TESTNAME: the name of the test, default: "yamllint" +# :type TESTNAME: string +# :param ARGN: the files or directories to check +# :type ARGN: list of strings +# +# @public +# +function(ament_yamllint) + cmake_parse_arguments(ARG "" "MAX_LINE_LENGTH;TESTNAME" "" ${ARGN}) + if(NOT ARG_TESTNAME) + set(ARG_TESTNAME "yamllint") + endif() + + find_program(ament_yamllint_BIN NAMES "ament_yamllint") + if(NOT ament_yamllint_BIN) + message(FATAL_ERROR "ament_yamllint() could not find program 'ament_yamllint'") + endif() + + set(result_file "${AMENT_TEST_RESULTS_DIR}/${PROJECT_NAME}/${ARG_TESTNAME}.xunit.xml") + set(cmd "${ament_yamllint_BIN}" "--xunit-file" "${result_file}") + list(APPEND cmd ${ARG_UNPARSED_ARGUMENTS}) + + find_program(yamllint_BIN NAMES "yamllint") + + if(NOT yamllint_BIN) + if(${CMAKE_VERSION} VERSION_LESS "3.8.0") + message(WARNING "WARNING: 'yamllint' not found, skipping yamllint test creation") + return() + endif() + endif() + + file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/ament_yamllint") + ament_add_test( + "${ARG_TESTNAME}" + COMMAND ${cmd} + OUTPUT_FILE "${CMAKE_BINARY_DIR}/ament_yamllint/${ARG_TESTNAME}.txt" + RESULT_FILE "${result_file}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) + set_tests_properties( + "${ARG_TESTNAME}" + PROPERTIES + LABELS "yamllint;linter" + ) + if(NOT yamllint_BIN) + set_tests_properties( + "${ARG_TESTNAME}" + PROPERTIES + DISABLED TRUE + ) + endif() +endfunction() diff --git a/ament_cmake_yamllint/doc/index.rst b/ament_cmake_yamllint/doc/index.rst new file mode 100644 index 000000000..9af022378 --- /dev/null +++ b/ament_cmake_yamllint/doc/index.rst @@ -0,0 +1,37 @@ +ament_yamllint +============== + +Checks YAML files using `yamllint `_. +Files with the following extensions are being considered: ``.yaml``, ``.yml``. + + +How to run the check from the command line? +------------------------------------------- + +The command line tool is provided by the package `ament_yamllint +`_. + + +How to run the check from within a CMake ament package as part of the tests? +---------------------------------------------------------------------------- + +``package.xml``: + +.. code:: xml + + ament_cmake + ament_cmake_yamllint + +``CMakeLists.txt``: + +.. code:: cmake + + find_package(ament_cmake REQUIRED) + if(BUILD_TESTING) + find_package(ament_cmake_yamllint REQUIRED) + ament_yamllint() + endif() + +The documentation of the package `ament_cmake_test +`_ provides more information on testing +in CMake ament packages. diff --git a/ament_cmake_yamllint/package.xml b/ament_cmake_yamllint/package.xml new file mode 100644 index 000000000..88e65e940 --- /dev/null +++ b/ament_cmake_yamllint/package.xml @@ -0,0 +1,30 @@ + + + + ament_cmake_yamllint + 0.11.3 + + The CMake API for ament_yamllint to check YAML file using yamllint. + + + Michael Jeronimo + Michel Hidalgo + + Apache License 2.0 + + Scott K Logan + + ament_cmake_core + ament_cmake_test + + ament_cmake_test + ament_yamllint + + ament_cmake_copyright + ament_cmake_lint_cmake + ament_cmake_xmllint + + + ament_cmake + + diff --git a/ament_yamllint/CHANGELOG.rst b/ament_yamllint/CHANGELOG.rst new file mode 100644 index 000000000..e69de29bb diff --git a/ament_yamllint/ament_yamllint/__init__.py b/ament_yamllint/ament_yamllint/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ament_yamllint/ament_yamllint/configuration/yamllint.yaml b/ament_yamllint/ament_yamllint/configuration/yamllint.yaml new file mode 100644 index 000000000..8a8bdd478 --- /dev/null +++ b/ament_yamllint/ament_yamllint/configuration/yamllint.yaml @@ -0,0 +1,9 @@ +--- +extends: default + +rules: + document-start: disable + indentation: + indent-sequences: consistent + line-length: disable + new-lines: disable diff --git a/ament_yamllint/ament_yamllint/main.py b/ament_yamllint/ament_yamllint/main.py new file mode 100755 index 000000000..6c90d296f --- /dev/null +++ b/ament_yamllint/ament_yamllint/main.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 + +# Copyright 2022 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import errno +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +from xml.sax.saxutils import escape +from xml.sax.saxutils import quoteattr + +import yaml + + +def main(argv=sys.argv[1:]): + config_file = os.path.join( + os.path.dirname(__file__), 'configuration', 'yamllint.yaml') + + extensions = ['yaml', 'yml'] + + parser = argparse.ArgumentParser( + description='Check YAML style using YAMLlint.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + '-c', '--config', + metavar='CFG', + default=config_file, + dest='config_file', + help='The config file') + parser.add_argument( + '--linelength', metavar='N', type=int, + help='The maximum line length (default: specified in the config file)') + parser.add_argument( + 'paths', + nargs='*', + default=[os.curdir], + help='The files or directories to check. For directories files ending ' + 'in %s will be considered.' % + ', '.join(["'.%s'" % e for e in extensions])) + parser.add_argument( + '--exclude', + nargs='*', + default=[], + help='Exclude specific file names and directory names from the check') + # not using a file handle directly + # in order to prevent leaving an empty file when something fails early + parser.add_argument( + '--xunit-file', + help='Generate a xunit compliant XML file') + args = parser.parse_args(argv) + + if not os.path.exists(args.config_file): + print("Could not config file '%s'" % args.config_file, file=sys.stderr) + return 1 + + files = get_files(args.paths, extensions, args.exclude) + if not files: + print('No files found', file=sys.stderr) + return 1 + + if args.xunit_file: + start_time = time.time() + + with open(args.config_file) as f: + yamllint_config = yaml.safe_load(f) + + assert isinstance(yamllint_config, dict), 'Invalid configuration file' + yamllint_config['yaml-files'] = ['*.%s' % (ext,) for ext in extensions] + if args.linelength is not None: + if not isinstance(yamllint_config.get('rules'), dict): + yamllint_config['rules'] = {} + if not isinstance(yamllint_config['rules'].get('line-length'), dict): + yamllint_config['rules']['line-length'] = {} + yamllint_config['rules']['line-length']['max'] = args.linelength + + temp_config_fd, temp_config_file = tempfile.mkstemp(suffix='.yaml', + prefix='yamllint_') + with os.fdopen(temp_config_fd, 'w') as f: + yaml.dump(yamllint_config, f) + + try: + report = invoke_yamllint(files, config_file=temp_config_file) + except: # noqa: E722 + raise + finally: + os.remove(temp_config_file) + + # generate xunit file + if args.xunit_file: + folder_name = os.path.basename(os.path.dirname(args.xunit_file)) + file_name = os.path.basename(args.xunit_file) + suffix = '.xml' + if file_name.endswith(suffix): + file_name = file_name[0:-len(suffix)] + suffix = '.xunit' + if file_name.endswith(suffix): + file_name = file_name[0:-len(suffix)] + testname = '%s.%s' % (folder_name, file_name) + + xml = get_xunit_content(report, testname, time.time() - start_time) + path = os.path.dirname(os.path.abspath(args.xunit_file)) + if not os.path.exists(path): + os.makedirs(path) + with open(args.xunit_file, 'w') as f: + f.write(xml) + + if any(report.values()): + return 1 + + print('No problems found, checked %d files' % (len(report),)) + return 0 + + +def get_files(paths, extensions, excludes=[]): + files = [] + for path in paths: + if os.path.isdir(path): + for dirpath, dirnames, filenames in os.walk(path): + if 'AMENT_IGNORE' in dirnames + filenames: + dirnames[:] = [] + continue + # ignore folder starting with . or _ + dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']] + # ignore excluded folders + dirnames[:] = [d for d in dirnames if d not in excludes] + dirnames.sort() + + # select files by extension + for filename in sorted(filenames): + if filename in excludes: + continue + _, ext = os.path.splitext(filename) + if ext not in ['.%s' % e for e in extensions]: + continue + files.append(os.path.join(dirpath, filename)) + if os.path.isfile(path): + files.append(path) + return [os.path.normpath(f) for f in files] + + +def find_executable(file_name, additional_paths=None): + path = None + if additional_paths: + path = os.getenv('PATH', os.defpath) + path += os.path.pathsep + os.path.pathsep.join(additional_paths) + return shutil.which(file_name, path=path) + + +def invoke_yamllint(files, yamllint_bin=None, config_file=None): + if yamllint_bin is None: + yamllint_bin = find_executable('yamllint') + if not yamllint_bin: + raise FileNotFoundError( + errno.ENOENT, 'Could not find executable', 'yamllint') + + cmd = [yamllint_bin, '-s', '-f', 'parsable'] + if config_file: + cmd += ['-c', config_file] + cmd += files + + report = {f: [] for f in files} + try: + subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + if e.stderr: + print(e.stderr.decode()) + stdout = e.stdout.decode() + print(stdout, end='') + if not stdout or e.returncode not in (1, 2): + raise + parser = re.compile(r'^(.+):(\d+):(\d+): \[(.+)\] (.*) \((.+)\)$') + for line in stdout.splitlines(): + m = parser.match(line) + if not m: + raise ValueError( + 'Failed to parse yamllint output: %s' % (line,)) + try: + report[m.group(1)].append({ + 'line': int(m.group(2)), + 'col': int(m.group(3)), + 'severity': m.group(4), + 'msg': m.group(5), + 'id': m.group(6), + }) + except NameError: + raise ValueError( + 'Got failures for unknown file: %s' % (m.group(1),)) + + return report + + +def get_xunit_content(report, testname, elapsed): + test_count = sum(max(len(r), 1) for r in report.values()) + error_count = sum(len(r) for r in report.values()) + data = { + 'testname': testname, + 'test_count': test_count, + 'error_count': error_count, + 'time': '%.3f' % round(elapsed, 3), + } + xml = """ + +""" % data + + for filename in sorted(report.keys()): + errors = report[filename] + + if errors: + # report each cppcheck error as a failing testcase + for error in errors: + data = { + 'quoted_name': quoteattr( + '%s: %s (%s:%d:%d)' % ( + error['severity'], error['id'], + filename, error['line'], error['col'])), + 'testname': testname, + 'quoted_message': quoteattr(error['msg']), + } + xml += """ + + +""" % data + + else: + # if there are no cpplint errors report a single successful test + data = { + 'quoted_location': quoteattr(filename), + 'testname': testname, + } + xml += """ +""" % data + + data = { + 'escaped_files': escape( + ''.join(['\n* %s' % r for r in sorted(report.keys())]) + ), + } + xml += """ Checked files:%(escaped_files)s +""" % data + + xml += '\n' + return xml + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ament_yamllint/ament_yamllint/pytest_marker.py b/ament_yamllint/ament_yamllint/pytest_marker.py new file mode 100644 index 000000000..7ebb5c89a --- /dev/null +++ b/ament_yamllint/ament_yamllint/pytest_marker.py @@ -0,0 +1,18 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def pytest_configure(config): + config.addinivalue_line( + 'markers', 'yamllint: marks tests checking for YAMLlint compliance') diff --git a/ament_yamllint/package.xml b/ament_yamllint/package.xml new file mode 100644 index 000000000..52a2875da --- /dev/null +++ b/ament_yamllint/package.xml @@ -0,0 +1,31 @@ + + + + ament_yamllint + 0.11.4 + + The ability to check YAML against style conventions using YAMLlint + and generate xUnit test result files. + + + Michael Jeronimo + Michel Hidalgo + + Apache License 2.0 + + Scott K Logan + + python3-yaml + yamllint + + ament_copyright + ament_flake8 + ament_pep257 + ament_pycodestyle + ament_xmllint + python3-pytest + + + ament_python + + diff --git a/ament_yamllint/pytest.ini b/ament_yamllint/pytest.ini new file mode 100644 index 000000000..fe55d2ed6 --- /dev/null +++ b/ament_yamllint/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +junit_family=xunit2 diff --git a/ament_yamllint/resource/ament_yamllint b/ament_yamllint/resource/ament_yamllint new file mode 100644 index 000000000..e69de29bb diff --git a/ament_yamllint/setup.py b/ament_yamllint/setup.py new file mode 100644 index 000000000..46f347016 --- /dev/null +++ b/ament_yamllint/setup.py @@ -0,0 +1,48 @@ +from setuptools import find_packages +from setuptools import setup + + +package_name = 'ament_yamllint' + +setup( + name=package_name, + version='0.11.4', + packages=find_packages(exclude=['test']), + data_files=[ + ('share/' + package_name, ['package.xml']), + ('share/ament_index/resource_index/packages', + ['resource/' + package_name]), + ], + install_requires=['PyYAML', 'setuptools'], + package_data={'': [ + 'configuration/yamllint.yaml', + ]}, + zip_safe=False, + author='Scott K Logan', + author_email='logans@cottsay.net', + maintainer='Michael Jeronimo, Michel Hidalgo', + maintainer_email='michael.jeronimo@openrobotics.org, michel@ekumenlabs.com', + url='https://github.com/ament/ament_lint', + download_url='https://github.com/ament/ament_lint/releases', + keywords=['ROS'], + classifiers=[ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Topic :: Software Development', + ], + description='Check YAML style using YAMLlint.', + long_description="""\ +The ability to check YAML against style conventions using YAMLlint +and generate xUnit test result files.""", + license='Apache License, Version 2.0', + tests_require=['pytest'], + entry_points={ + 'console_scripts': [ + 'ament_yamllint = ament_yamllint.main:main', + ], + 'pytest11': [ + 'ament_yamllint = ament_yamllint.pytest_marker', + ], + }, +) diff --git a/ament_yamllint/test/test_copyright.py b/ament_yamllint/test/test_copyright.py new file mode 100644 index 000000000..4962484ea --- /dev/null +++ b/ament_yamllint/test/test_copyright.py @@ -0,0 +1,23 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_copyright.main import main +import pytest + + +@pytest.mark.copyright +@pytest.mark.linter +def test_copyright(): + rc = main(argv=['.', 'test']) + assert rc == 0, 'Found errors' diff --git a/ament_yamllint/test/test_flake8.py b/ament_yamllint/test/test_flake8.py new file mode 100644 index 000000000..4d62a9ca2 --- /dev/null +++ b/ament_yamllint/test/test_flake8.py @@ -0,0 +1,25 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_flake8.main import main_with_errors +import pytest + + +@pytest.mark.flake8 +@pytest.mark.linter +def test_flake8(): + rc, errors = main_with_errors(argv=[]) + assert rc == 0, \ + 'Found %d code style errors / warnings:\n' % len(errors) + \ + '\n'.join(errors) diff --git a/ament_yamllint/test/test_pep257.py b/ament_yamllint/test/test_pep257.py new file mode 100644 index 000000000..f7ef98bbe --- /dev/null +++ b/ament_yamllint/test/test_pep257.py @@ -0,0 +1,23 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_pep257.main import main +import pytest + + +@pytest.mark.linter +@pytest.mark.pep257 +def test_pep257(): + rc = main(argv=['.', 'test']) + assert rc == 0, 'Found code style errors / warnings' diff --git a/ament_yamllint/test/test_pycodestyle.py b/ament_yamllint/test/test_pycodestyle.py new file mode 100644 index 000000000..d0b4be08b --- /dev/null +++ b/ament_yamllint/test/test_pycodestyle.py @@ -0,0 +1,22 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_pycodestyle.main import main +import pytest + + +@pytest.mark.linter +def test_pycodestyle(): + rc = main(argv=['.', 'test']) + assert rc == 0, 'Found code style errors / warnings' diff --git a/ament_yamllint/test/test_xmllint.py b/ament_yamllint/test/test_xmllint.py new file mode 100644 index 000000000..8fea40215 --- /dev/null +++ b/ament_yamllint/test/test_xmllint.py @@ -0,0 +1,23 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_xmllint.main import main +import pytest + + +@pytest.mark.linter +@pytest.mark.xmllint +def test_xmllint(): + rc = main(argv=[]) + assert rc == 0, 'Found XML style errors / warnings' diff --git a/ament_yamllint/test/test_yamllint.py b/ament_yamllint/test/test_yamllint.py new file mode 100644 index 000000000..a0db16c3d --- /dev/null +++ b/ament_yamllint/test/test_yamllint.py @@ -0,0 +1,23 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_yamllint.main import main +import pytest + + +@pytest.mark.linter +@pytest.mark.yamllint +def test_yamllint(): + rc = main(argv=[]) + assert rc == 0, 'Found YAML style errors / warnings'