diff --git a/.github/workflows/ruby-tests.yml b/.github/workflows/ruby-tests.yml index 68a6e25..a7f3722 100644 --- a/.github/workflows/ruby-tests.yml +++ b/.github/workflows/ruby-tests.yml @@ -27,3 +27,6 @@ jobs: - name: Run tests run: bundle exec rspec + + - name: Run rubocop + run: bundle exec rubocop diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..8c3e2b1 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,35 @@ +inherit_from: .rubocop_todo.yml + +AllCops: + NewCops: disable + SuggestExtensions: false + TargetRubyVersion: 3.0 + +Layout/LineLength: + Max: 199 + +Metrics/BlockLength: + Max: 63 + +Metrics/ClassLength: + Max: 200 + +Metrics/CyclomaticComplexity: + Max: 8 + +Metrics/MethodLength: + Max: 20 + +Style/ClassVars: + Exclude: + - 'lib/earl/scraper.rb' + +Style/Documentation: + Exclude: + - 'spec/**/*' + +Style/ClassAndModuleChildren: + EnforcedStyleForClasses: compact + +Layout/FirstHashElementIndentation: + EnforcedStyle: consistent diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml new file mode 100644 index 0000000..40e8b4a --- /dev/null +++ b/.rubocop_todo.yml @@ -0,0 +1,22 @@ +# This configuration was generated by +# `rubocop --auto-gen-config` +# on 2025-07-29 10:15:32 UTC using RuboCop version 1.79.0. +# The point is for the user to remove these configuration records +# one by one as the offenses are removed from the code base. +# Note that changes in the inspected code, or installation of new +# versions of RuboCop, may require this file to be generated again. + +# Offense count: 7 +# This cop supports unsafe autocorrection (--autocorrect-all). +# Configuration parameters: AllowSafeAssignment. +Lint/AssignmentInCondition: + Exclude: + - 'lib/earl/earl.rb' + - 'lib/earl/scraper.rb' + +# Offense count: 2 +# Configuration parameters: AllowedMethods. +# AllowedMethods: enums +Lint/ConstantDefinitionInBlock: + Exclude: + - 'spec/unit/earl/scraper_spec.rb' diff --git a/Gemfile b/Gemfile index d435dd3..799162a 100644 --- a/Gemfile +++ b/Gemfile @@ -8,7 +8,9 @@ gemspec # development dependencies gem 'bundler', '>= 2.2.33' gem 'guard-rspec' +gem 'guard-rubocop' gem 'rake' gem 'rspec' +gem 'rubocop', require: false gem 'vcr' gem 'webmock' diff --git a/Guardfile b/Guardfile index a889f69..8a3e5c9 100644 --- a/Guardfile +++ b/Guardfile @@ -1,8 +1,15 @@ +# frozen_string_literal: true + # A sample Guardfile # More info at https://github.com/guard/guard#readme guard :rspec, cmd: 'bundle exec rspec' do watch(%r{^spec/.+_spec\.rb$}) - watch('spec/spec_helper.rb') { "spec" } + watch('spec/spec_helper.rb') { 'spec' } watch(%r{^lib/(.+)\.rb$}) { |m| "spec/unit/#{m[1]}_spec.rb" } end + +guard :rubocop, cli: ['--display-cop-names'] do + watch(/.+\.rb$/) + watch(%r{(?:.+/)?\.rubocop(?:_todo)?\.yml$}) { |m| File.dirname(m[0]) } +end diff --git a/Rakefile b/Rakefile index 1d15da4..1715149 100644 --- a/Rakefile +++ b/Rakefile @@ -7,7 +7,7 @@ RSpec::Core::RakeTask.new(:spec) task default: :spec -desc "Open an irb session preloaded with this library" +desc 'Open an irb session preloaded with this library' task :console do - sh "irb -I lib -r earl.rb" + sh 'irb -I lib -r earl.rb' end diff --git a/earl.gemspec b/earl.gemspec index b9dd2f7..ebe2e73 100644 --- a/earl.gemspec +++ b/earl.gemspec @@ -1,21 +1,22 @@ # frozen_string_literal: true +require 'English' lib = File.expand_path('lib', __dir__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) require 'earl/version' Gem::Specification.new do |gem| - gem.authors = ["teejayvanslyke", "Paul Gallagher"] - gem.email = ["tj@elctech.com", "gallagher.paul@gmail.com"] + gem.authors = ['teejayvanslyke', 'Paul Gallagher'] + gem.email = ['tj@elctech.com', 'gallagher.paul@gmail.com'] gem.description = 'URL metadata API' gem.summary = 'URL metadata API for scraping titles, descriptions, images, and videos from URLs' gem.homepage = 'https://github.com/evendis/earl' - gem.files = `git ls-files`.split($\) - gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } + gem.files = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR) + gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) } gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.name = 'earl' - gem.require_paths = ["lib"] + gem.require_paths = ['lib'] gem.version = Earl::VERSION gem.license = 'MIT' diff --git a/lib/earl.rb b/lib/earl.rb index a758cee..3c138c2 100644 --- a/lib/earl.rb +++ b/lib/earl.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'nokogiri' require 'open-uri' diff --git a/lib/earl/earl.rb b/lib/earl/earl.rb index 5d7434d..51b762d 100644 --- a/lib/earl/earl.rb +++ b/lib/earl/earl.rb @@ -1,10 +1,14 @@ +# frozen_string_literal: true + require 'open-uri' require 'oembedr' +# Earl is a class that represents a URL and provides methods to fetch metadata about the page class Earl - attr_accessor :url, :options, :oembed + attr_accessor :url, :options + attr_writer :oembed - def initialize(url, options={}) + def initialize(url, options = {}) @url = url @options = options end @@ -18,7 +22,7 @@ def uri end def uri_response - @uri_response ||= URI.open(uri) + @uri_response ||= uri.open end # Returns @@ -35,13 +39,13 @@ def uri_response_attribute(name) when :headers uri_response_attribute(:meta) else - uri_response && uri_response.respond_to?(name) && uri_response.send(name) + uri_response.respond_to?(name) && uri_response.send(name) end end protected :uri_response_attribute def uri_response_attributes - [:content_type,:base_url,:charset,:content_encoding,:headers] + %i[content_type base_url charset content_encoding headers] end protected :uri_response_attributes @@ -50,7 +54,7 @@ def scraper end def response - scraper && scraper.response + scraper&.response end # Returns a hash of link meta data, including: @@ -59,23 +63,28 @@ def response def metadata data = oembed || {} attributes.each do |attribute| - if attribute_value = self.send(attribute) + if attribute_value = send(attribute) data[attribute] ||= attribute_value end end data end + def respond_to_missing?(name, include_private) + uri_response_attributes.include?(name) || scraper&.attribute?(name) || super + end + # Dispatch missing methods if a match for: # - uri_response_attributes # - scraper attributes def method_missing(method, *args) if uri_response_attributes.include?(method) - return uri_response_attribute(method) - elsif scraper && scraper.has_attribute?(method) - return scraper.attribute(method) + uri_response_attribute(method) + elsif scraper&.attribute?(method) + scraper.attribute(method) + else + super end - super end # Returns a full array of attributes available for the link @@ -85,7 +94,7 @@ def attributes # Returns the options to be used for oembed def oembed_options - { :maxwidth => "560", :maxheight => "315" }.merge(options[:oembed]||{}) + { maxwidth: '560', maxheight: '315' }.merge(options[:oembed] || {}) end # Returns the oembed meta data hash for the URL (or nil if not defined/available) e.g. @@ -108,21 +117,27 @@ def oembed_options # # +options+ defines a custom oembed options hash and will cause a re-fetch of the oembed metadata # TODO: Oembedr is outdated and not longer works with most/all providers - def oembed(options=nil) + def oembed(options = nil) if options # use custom options, refetch oembed metadata @options[:oembed] = options @oembed = nil end - begin - @oembed ||= if h = Oembedr.fetch(base_url, :params => oembed_options).body + @oembed ||= begin + h = Oembedr.fetch(base_url, params: oembed_options).body + if h h.keys.each do |key| # symbolize_keys! - h[(key.to_sym rescue key) || key] = h.delete(key) + new_key = begin + key.to_sym + rescue StandardError + key + end + h[new_key] = h.delete(key) end h end - rescue + rescue StandardError + nil end - @oembed end # Returns the oembed code for the url (or nil if not defined/available) @@ -131,7 +146,7 @@ def oembed_html end # Returns true if there is an ATOM or RSS feed associated with this URL. - def has_feed? + def feed? !feed.nil? end diff --git a/lib/earl/scraper.rb b/lib/earl/scraper.rb index 042b9dc..c8127a6 100644 --- a/lib/earl/scraper.rb +++ b/lib/earl/scraper.rb @@ -1,9 +1,10 @@ -class Earl::Scraper +# frozen_string_literal: true +# Base class for nokogiri page scraping +class Earl::Scraper class << self @@registry = [] - attr_reader :regexp - attr_reader :attributes + attr_reader :regexp, :attributes def match(regexp) @regexp = regexp @@ -17,9 +18,9 @@ def define_attribute(name, &block) def for(url, earl_source) @@registry.each do |klass| - return klass.new(url,earl_source) if klass.regexp.match(url) + return klass.new(url, earl_source) if klass.regexp.match(url) end - Earl::Scraper.new(url,earl_source) + Earl::Scraper.new(url, earl_source) end def register(scraper_klass) @@ -40,9 +41,9 @@ def response end def attribute(name) - return unless has_attribute?(name) + return unless attribute?(name) - self.attributes[name].call(response) + attributes[name].call(response) end def attributes @@ -53,10 +54,10 @@ def attributes end end - def has_attribute?(name) + def attribute?(name) return false unless self.class.attributes - self.attributes.has_key?(name) + attributes.key?(name) end define_attribute :title do |doc| diff --git a/lib/earl/version.rb b/lib/earl/version.rb index 6f4e7d7..4938bcf 100644 --- a/lib/earl/version.rb +++ b/lib/earl/version.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class Earl - VERSION = "1.0.0" + VERSION = '1.0.0' end diff --git a/spec/integration/feed_spec.rb b/spec/integration/feed_spec.rb index ce2c6af..28eb820 100644 --- a/spec/integration/feed_spec.rb +++ b/spec/integration/feed_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' describe Earl do @@ -7,69 +9,69 @@ context 'when page has no feeds associated', vcr: { cassette_name: "#{vcr_base}/no_feed" } do let(:url) { 'https://www.google.com/' } - it { expect(subject.has_feed?).to be false } + it { expect(subject.feed?).to be false } it { expect(subject.rss_feed).to be_nil } it { expect(subject.feed).to be_nil } end - context "when page has rss feed associated", vcr: { cassette_name: "#{vcr_base}/with_rss_feed" } do + context 'when page has rss feed associated', vcr: { cassette_name: "#{vcr_base}/with_rss_feed" } do let(:url) { 'https://rubyweekly.com/' } - it { expect(subject.has_feed?).to be true } + it { expect(subject.feed?).to be true } it { expect(subject.rss_feed).to eql('/rss/') } it { expect(subject.feed).to eql('/rss/') } end # # context "when page has atom feed associated" do # # let(:url) { 'http:// still looking for a page that just has atom' } - # # { expect(subject.has_feed?).to be true } + # # { expect(subject.feed?).to be true } # # { expect(subject.atom_feed).to eql('http://www.readwriteweb.com/atom.xml') } # # { expect(subject.feed).to eql('http://www.readwriteweb.com/atom.xml') } # # end - context "when page has rss and atom feed associated", vcr: { cassette_name: "#{vcr_base}/with_atom_and_rss_feed" } do + context 'when page has rss and atom feed associated', vcr: { cassette_name: "#{vcr_base}/with_atom_and_rss_feed" } do let(:url) { 'https://0xfe.blogspot.com/' } let(:expected_rss_feed) { 'https://0xfe.blogspot.com/feeds/posts/default?alt=rss' } let(:expected_atom_feed) { 'https://0xfe.blogspot.com/feeds/posts/default' } - it { expect(subject.has_feed?).to be true } + it { expect(subject.feed?).to be true } it { expect(subject.rss_feed).to eql(expected_rss_feed) } it { expect(subject.atom_feed).to eql(expected_atom_feed) } - describe "#feed" do - context "default (rss)" do + describe '#feed' do + context 'default (rss)' do subject { instance.feed } it { expect(subject).to eql(expected_rss_feed) } end - context "rss prefered" do + context 'rss prefered' do subject { instance.feed(:rss) } it { expect(subject).to eql(expected_rss_feed) } end - context "atom prefered" do + context 'atom prefered' do subject { instance.feed(:atom) } it { expect(subject).to eql(expected_atom_feed) } end end end - context "when page IS an rss feed", vcr: { cassette_name: "#{vcr_base}/is_rss_feed" } do + context 'when page IS an rss feed', vcr: { cassette_name: "#{vcr_base}/is_rss_feed" } do let(:url) { 'https://cprss.s3.amazonaws.com/rubyweekly.com.xml' } it { expect(subject.url).to eql(url) } it { expect(subject.base_url).to eql(url) } it { expect(subject.content_type).to eql('application/xml') } # TODO: fix rss feed detection - # it { expect(subject.has_feed?).to be true } + # it { expect(subject.feed?).to be true } # it { expect(subject.rss_feed).to eql(url) } # it { expect(subject.feed).to eql(url) } - it { expect(subject.has_feed?).to be false } + it { expect(subject.feed?).to be false } it { expect(subject.rss_feed).to be_nil } it { expect(subject.feed).to be_nil } end - context "when page IS an atom feed", vcr: { cassette_name: "#{vcr_base}/is_atom_feed" } do + context 'when page IS an atom feed', vcr: { cassette_name: "#{vcr_base}/is_atom_feed" } do let(:url) { 'https://0xfe.blogspot.com/feeds/posts/default' } it { expect(subject.url).to eql(url) } it { expect(subject.base_url).to eql(url) } it { expect(subject.content_type).to eql('application/atom+xml') } - it { expect(subject.has_feed?).to be true } + it { expect(subject.feed?).to be true } it { expect(subject.atom_feed).to eql(url) } it { expect(subject.feed).to eql(url) } end diff --git a/spec/integration/oembed_spec.rb b/spec/integration/oembed_spec.rb index 8c16954..cdd2f09 100644 --- a/spec/integration/oembed_spec.rb +++ b/spec/integration/oembed_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' describe Earl do @@ -5,14 +7,14 @@ vcr_base = 'feed' - context "when page does not support oembed", vcr: { cassette_name: "#{vcr_base}/no_oembed" } do + context 'when page does not support oembed', vcr: { cassette_name: "#{vcr_base}/no_oembed" } do let(:url) { 'https://github.com/evendis/earl' } - it { expect(subject.oembed).to eql({error: 'no matching providers found', url: 'https://github.com/evendis/earl'}) } + it { expect(subject.oembed).to eql({ error: 'no matching providers found', url: 'https://github.com/evendis/earl' }) } it { expect(subject.oembed_html).to be_nil } - describe "#metadata" do + describe '#metadata' do subject { instance.metadata } - it { expect(subject[:base_url]).to match(/github\.com\/evendis\/earl/) } - it { expect(subject[:content_type]).to eql("text/html") } + it { expect(subject[:base_url]).to match(%r{github\.com/evendis/earl}) } + it { expect(subject[:content_type]).to eql('text/html') } it { expect(subject[:html]).to be_nil } end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 7e138c3..c485e6d 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + require 'earl' # Requires supporting files with custom matchers and macros, etc, # in ./support/ and its subdirectories. -Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f} +Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f } RSpec.configure do |config| # == Mock Framework @@ -21,4 +23,4 @@ # examples within a transaction, remove the following line or assign false # instead of true. # config.use_transactional_fixtures = true -end \ No newline at end of file +end diff --git a/spec/support/fixtures.rb b/spec/support/fixtures.rb index 2fe13e7..8eb3ed1 100644 --- a/spec/support/fixtures.rb +++ b/spec/support/fixtures.rb @@ -1,10 +1,15 @@ +# frozen_string_literal: true + module FixtureHelper - def fixture(name) - # File.new( File.dirname(__FILE__) + "/../fixtures/#{name}.html" ).read + def fixture_path(name) File.dirname(__FILE__) + "/../fixtures/#{name}.html" end + + def fixture(name) + File.new(fixture_path(name)).read + end end RSpec.configure do |conf| conf.include FixtureHelper -end \ No newline at end of file +end diff --git a/spec/support/vcr.rb b/spec/support/vcr.rb index bdb41f6..0d68d13 100644 --- a/spec/support/vcr.rb +++ b/spec/support/vcr.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'vcr' VCR.configure do |config| diff --git a/spec/unit/earl/earl_spec.rb b/spec/unit/earl/earl_spec.rb index 35dd319..173e260 100644 --- a/spec/unit/earl/earl_spec.rb +++ b/spec/unit/earl/earl_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' describe Earl do diff --git a/spec/unit/earl/feed_spec.rb b/spec/unit/earl/feed_spec.rb index 5d8be90..834af90 100644 --- a/spec/unit/earl/feed_spec.rb +++ b/spec/unit/earl/feed_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' describe Earl do @@ -5,46 +7,46 @@ let(:url) { 'http:://example.com' } before do - allow_any_instance_of(Earl).to receive(:uri).and_return(source) + allow_any_instance_of(Earl).to receive(:uri_response).and_return(source) end subject { instance } - context "when page has no feeds associated" do + context 'when page has no feeds associated' do let(:source) { fixture(:page_without_feeds) } - it { expect(subject.has_feed?).to be false } + it { expect(subject.feed?).to be false } it { expect(subject.rss_feed).to be_nil } it { expect(subject.feed).to be_nil } end - context "when page has rss feed associated" do + context 'when page has rss feed associated' do let(:source) { fixture(:page_with_rss_feed) } - it { expect(subject.has_feed?).to be true } + it { expect(subject.feed?).to be true } it { expect(subject.rss_feed).to eql('http://www.readwriteweb.com/rss.xml') } it { expect(subject.feed).to eql('http://www.readwriteweb.com/rss.xml') } end - context "when page has atom feed associated" do + context 'when page has atom feed associated' do let(:source) { fixture(:page_with_atom_feed) } - it { expect(subject.has_feed?).to be true } + it { expect(subject.feed?).to be true } it { expect(subject.atom_feed).to eql('http://www.readwriteweb.com/atom.xml') } it { expect(subject.feed).to eql('http://www.readwriteweb.com/atom.xml') } end - context "when page has rss and atom feed associated" do + context 'when page has rss and atom feed associated' do let(:source) { fixture(:page_with_rss_and_atom_feeds) } - it { expect(subject.has_feed?).to be true } + it { expect(subject.feed?).to be true } it { expect(subject.rss_feed).to eql('http://www.readwriteweb.com/rss.xml') } it { expect(subject.atom_feed).to eql('http://www.readwriteweb.com/atom.xml') } - describe "#feed" do - context "default (rss)" do + describe '#feed' do + context 'default (rss)' do subject { instance.feed } it { expect(subject).to eql('http://www.readwriteweb.com/rss.xml') } end - context "rss prefered" do + context 'rss prefered' do subject { instance.feed(:rss) } it { expect(subject).to eql('http://www.readwriteweb.com/rss.xml') } end - context "atom prefered" do + context 'atom prefered' do subject { instance.feed(:atom) } it { expect(subject).to eql('http://www.readwriteweb.com/atom.xml') } end @@ -53,7 +55,7 @@ # context "when page IS an rss feed" do # let(:source) { fixture(:page_as_rss) } - # its(:has_feed?) { should be_true } + # its(:feed?) { should be_true } # its(:rss_feed) { should eql(url) } # its(:feed) { should eql(url) } # end diff --git a/spec/unit/earl/oembed_spec.rb b/spec/unit/earl/oembed_spec.rb index 298dcf3..28955b3 100644 --- a/spec/unit/earl/oembed_spec.rb +++ b/spec/unit/earl/oembed_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' class MockOembedResponse @@ -7,40 +9,40 @@ def body end describe Earl do - let(:instance) { Earl.new(url,options) } + let(:instance) { Earl.new(url, options) } let(:url) { 'http:://example.com' } let(:options) { {} } - describe "#oembed_options" do + describe '#oembed_options' do subject { instance.oembed_options } - context "with default options" do - let(:expected) { { :maxwidth => "560", :maxheight => "315" } } + context 'with default options' do + let(:expected) { { maxwidth: '560', maxheight: '315' } } it { expect(subject).to eql(expected) } end - context "with custom options" do - let(:options) { { :oembed => { :maxwidth => "260" } } } - let(:expected) { { :maxwidth => "260", :maxheight => "315" } } + context 'with custom options' do + let(:options) { { oembed: { maxwidth: '260' } } } + let(:expected) { { maxwidth: '260', maxheight: '315' } } it { expect(subject).to eql(expected) } end - context "with custom options passed to oembed" do - let(:expected) { { :maxwidth => "360", :maxheight => "315" } } + context 'with custom options passed to oembed' do + let(:expected) { { maxwidth: '360', maxheight: '315' } } before do allow(Oembedr).to receive(:fetch).and_return(nil) - instance.oembed({ :maxwidth => "360" }) + instance.oembed({ maxwidth: '360' }) end it { expect(subject).to eql(expected) } end end - describe "#oembed" do + describe '#oembed' do let(:dummy_response) { MockOembedResponse.new } before do expect(instance).to receive(:base_url).and_return('') expect(Oembedr).to receive(:fetch).and_return(dummy_response) end subject { instance.oembed } - it { expect(subject).to eql({ :html => '