Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/ruby-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,6 @@ jobs:

- name: Run tests
run: bundle exec rspec

- name: Run rubocop
run: bundle exec rubocop
35 changes: 35 additions & 0 deletions .rubocop.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
inherit_from: .rubocop_todo.yml

AllCops:
NewCops: disable
SuggestExtensions: false
TargetRubyVersion: 3.0

Layout/LineLength:
Max: 199

Metrics/BlockLength:
Max: 63

Metrics/ClassLength:
Max: 200

Metrics/CyclomaticComplexity:
Max: 8

Metrics/MethodLength:
Max: 20

Style/ClassVars:
Exclude:
- 'lib/earl/scraper.rb'

Style/Documentation:
Exclude:
- 'spec/**/*'

Style/ClassAndModuleChildren:
EnforcedStyleForClasses: compact

Layout/FirstHashElementIndentation:
EnforcedStyle: consistent
22 changes: 22 additions & 0 deletions .rubocop_todo.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# This configuration was generated by
# `rubocop --auto-gen-config`
# on 2025-07-29 10:15:32 UTC using RuboCop version 1.79.0.
# The point is for the user to remove these configuration records
# one by one as the offenses are removed from the code base.
# Note that changes in the inspected code, or installation of new
# versions of RuboCop, may require this file to be generated again.

# Offense count: 7
# This cop supports unsafe autocorrection (--autocorrect-all).
# Configuration parameters: AllowSafeAssignment.
Lint/AssignmentInCondition:
Exclude:
- 'lib/earl/earl.rb'
- 'lib/earl/scraper.rb'

# Offense count: 2
# Configuration parameters: AllowedMethods.
# AllowedMethods: enums
Lint/ConstantDefinitionInBlock:
Exclude:
- 'spec/unit/earl/scraper_spec.rb'
2 changes: 2 additions & 0 deletions Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ gemspec
# development dependencies
gem 'bundler', '>= 2.2.33'
gem 'guard-rspec'
gem 'guard-rubocop'
gem 'rake'
gem 'rspec'
gem 'rubocop', require: false
gem 'vcr'
gem 'webmock'
9 changes: 8 additions & 1 deletion Guardfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
# frozen_string_literal: true

# A sample Guardfile
# More info at https://github.com/guard/guard#readme

guard :rspec, cmd: 'bundle exec rspec' do
watch(%r{^spec/.+_spec\.rb$})
watch('spec/spec_helper.rb') { "spec" }
watch('spec/spec_helper.rb') { 'spec' }
watch(%r{^lib/(.+)\.rb$}) { |m| "spec/unit/#{m[1]}_spec.rb" }
end

guard :rubocop, cli: ['--display-cop-names'] do
watch(/.+\.rb$/)
watch(%r{(?:.+/)?\.rubocop(?:_todo)?\.yml$}) { |m| File.dirname(m[0]) }
end
4 changes: 2 additions & 2 deletions Rakefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ RSpec::Core::RakeTask.new(:spec)

task default: :spec

desc "Open an irb session preloaded with this library"
desc 'Open an irb session preloaded with this library'
task :console do
sh "irb -I lib -r earl.rb"
sh 'irb -I lib -r earl.rb'
end
11 changes: 6 additions & 5 deletions earl.gemspec
Original file line number Diff line number Diff line change
@@ -1,21 +1,22 @@
# frozen_string_literal: true

require 'English'
lib = File.expand_path('lib', __dir__)
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
require 'earl/version'

Gem::Specification.new do |gem|
gem.authors = ["teejayvanslyke", "Paul Gallagher"]
gem.email = ["tj@elctech.com", "gallagher.paul@gmail.com"]
gem.authors = ['teejayvanslyke', 'Paul Gallagher']
gem.email = ['tj@elctech.com', 'gallagher.paul@gmail.com']
gem.description = 'URL metadata API'
gem.summary = 'URL metadata API for scraping titles, descriptions, images, and videos from URLs'
gem.homepage = 'https://github.com/evendis/earl'

gem.files = `git ls-files`.split($\)
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
gem.files = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR)
gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
gem.name = 'earl'
gem.require_paths = ["lib"]
gem.require_paths = ['lib']
gem.version = Earl::VERSION
gem.license = 'MIT'

Expand Down
2 changes: 2 additions & 0 deletions lib/earl.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'nokogiri'
require 'open-uri'

Expand Down
53 changes: 34 additions & 19 deletions lib/earl/earl.rb
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
# frozen_string_literal: true

require 'open-uri'
require 'oembedr'

# Earl is a class that represents a URL and provides methods to fetch metadata about the page
class Earl
attr_accessor :url, :options, :oembed
attr_accessor :url, :options
attr_writer :oembed

def initialize(url, options={})
def initialize(url, options = {})
@url = url
@options = options
end
Expand All @@ -18,7 +22,7 @@ def uri
end

def uri_response
@uri_response ||= URI.open(uri)
@uri_response ||= uri.open
end

# Returns
Expand All @@ -35,13 +39,13 @@ def uri_response_attribute(name)
when :headers
uri_response_attribute(:meta)
else
uri_response && uri_response.respond_to?(name) && uri_response.send(name)
uri_response.respond_to?(name) && uri_response.send(name)
end
end
protected :uri_response_attribute

def uri_response_attributes
[:content_type,:base_url,:charset,:content_encoding,:headers]
%i[content_type base_url charset content_encoding headers]
end
protected :uri_response_attributes

Expand All @@ -50,7 +54,7 @@ def scraper
end

def response
scraper && scraper.response
scraper&.response
end

# Returns a hash of link meta data, including:
Expand All @@ -59,23 +63,28 @@ def response
def metadata
data = oembed || {}
attributes.each do |attribute|
if attribute_value = self.send(attribute)
if attribute_value = send(attribute)
data[attribute] ||= attribute_value
end
end
data
end

def respond_to_missing?(name, include_private)
uri_response_attributes.include?(name) || scraper&.attribute?(name) || super
end

# Dispatch missing methods if a match for:
# - uri_response_attributes
# - scraper attributes
def method_missing(method, *args)
if uri_response_attributes.include?(method)
return uri_response_attribute(method)
elsif scraper && scraper.has_attribute?(method)
return scraper.attribute(method)
uri_response_attribute(method)
elsif scraper&.attribute?(method)
scraper.attribute(method)
else
super
end
super
end

# Returns a full array of attributes available for the link
Expand All @@ -85,7 +94,7 @@ def attributes

# Returns the options to be used for oembed
def oembed_options
{ :maxwidth => "560", :maxheight => "315" }.merge(options[:oembed]||{})
{ maxwidth: '560', maxheight: '315' }.merge(options[:oembed] || {})
end

# Returns the oembed meta data hash for the URL (or nil if not defined/available) e.g.
Expand All @@ -108,21 +117,27 @@ def oembed_options
#
# +options+ defines a custom oembed options hash and will cause a re-fetch of the oembed metadata
# TODO: Oembedr is outdated and not longer works with most/all providers
def oembed(options=nil)
def oembed(options = nil)
if options # use custom options, refetch oembed metadata
@options[:oembed] = options
@oembed = nil
end
begin
@oembed ||= if h = Oembedr.fetch(base_url, :params => oembed_options).body
@oembed ||= begin
h = Oembedr.fetch(base_url, params: oembed_options).body
if h
h.keys.each do |key| # symbolize_keys!
h[(key.to_sym rescue key) || key] = h.delete(key)
new_key = begin
key.to_sym
rescue StandardError
key
end
h[new_key] = h.delete(key)
end
h
end
rescue
rescue StandardError
nil
end
@oembed
end

# Returns the oembed code for the url (or nil if not defined/available)
Expand All @@ -131,7 +146,7 @@ def oembed_html
end

# Returns true if there is an ATOM or RSS feed associated with this URL.
def has_feed?
def feed?
!feed.nil?
end

Expand Down
19 changes: 10 additions & 9 deletions lib/earl/scraper.rb
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
class Earl::Scraper
# frozen_string_literal: true

# Base class for nokogiri page scraping
class Earl::Scraper
class << self
@@registry = []
attr_reader :regexp
attr_reader :attributes
attr_reader :regexp, :attributes

def match(regexp)
@regexp = regexp
Expand All @@ -17,9 +18,9 @@ def define_attribute(name, &block)

def for(url, earl_source)
@@registry.each do |klass|
return klass.new(url,earl_source) if klass.regexp.match(url)
return klass.new(url, earl_source) if klass.regexp.match(url)
end
Earl::Scraper.new(url,earl_source)
Earl::Scraper.new(url, earl_source)
end

def register(scraper_klass)
Expand All @@ -40,9 +41,9 @@ def response
end

def attribute(name)
return unless has_attribute?(name)
return unless attribute?(name)

self.attributes[name].call(response)
attributes[name].call(response)
end

def attributes
Expand All @@ -53,10 +54,10 @@ def attributes
end
end

def has_attribute?(name)
def attribute?(name)
return false unless self.class.attributes

self.attributes.has_key?(name)
attributes.key?(name)
end

define_attribute :title do |doc|
Expand Down
4 changes: 3 additions & 1 deletion lib/earl/version.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

class Earl
VERSION = "1.0.0"
VERSION = '1.0.0'
end
Loading