From 6ab46c1248005483d9cc4d610c8c543c14b51e6a Mon Sep 17 00:00:00 2001 From: Sylvain Joyeux Date: Fri, 18 Dec 2015 10:31:48 -0200 Subject: [PATCH 1/8] timepoints: use 'start' and 'done' as keywords at the end of a timepoint name to compute a total duration --- lib/utilrb/timepoints.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/utilrb/timepoints.rb b/lib/utilrb/timepoints.rb index c7fc58f..5597025 100644 --- a/lib/utilrb/timepoints.rb +++ b/lib/utilrb/timepoints.rb @@ -15,8 +15,15 @@ def add_timepoint(*names) end def format_timepoints + start_points = Hash.new result = [] @timepoints.inject(@timepoints.first.first) do |last_t, (t, name)| + if name.last == 'start' + start_points[name[0..-2]] = t + elsif name.last == 'done' + total = t - start_points.delete(name[0..-2]) + name = name + ["total=%.3f" % total] + end result << name + [t - last_t] t end From 6501cf38023e899e4a8f9cd03f11808e712b3343 Mon Sep 17 00:00:00 2001 From: Sylvain Joyeux Date: Mon, 18 Apr 2016 11:04:13 -0300 Subject: [PATCH 2/8] marshal: improve robustness of load_with_missing_constants --- lib/utilrb/marshal.rb | 3 ++ .../marshal/load_with_missing_constants.rb | 39 +++++++-------- test/test_marshal.rb | 47 +++++++++++++++++++ 3 files changed, 68 insertions(+), 21 deletions(-) create mode 100644 lib/utilrb/marshal.rb create mode 100644 test/test_marshal.rb diff --git a/lib/utilrb/marshal.rb b/lib/utilrb/marshal.rb new file mode 100644 index 0000000..738bf8c --- /dev/null +++ b/lib/utilrb/marshal.rb @@ -0,0 +1,3 @@ +require 'utilrb/kernel/require' +require_dir(__FILE__) + diff --git a/lib/utilrb/marshal/load_with_missing_constants.rb b/lib/utilrb/marshal/load_with_missing_constants.rb index f2a7752..39a69fb 100644 --- a/lib/utilrb/marshal/load_with_missing_constants.rb +++ b/lib/utilrb/marshal/load_with_missing_constants.rb @@ -1,26 +1,32 @@ module Marshal - if defined? BasicObject - class BlackHole < BasicObject - end - end - class BlackHole class << self - :name + attr_reader :name end def initialize(*args) end + def hash + __id__ + end + + def eql?(obj) + equal?(obj) + end + attr_reader :__content__ def method_missing(*args) + ::Kernel.puts args.inspect + ::Kernel.puts ::Kernel.caller end def self._load(*args) hole = BlackHole.new hole.instance_variable_set(:@__content__, args) end - def self.method_missing(*args) + def self.method_missing(*args, **options) + BlackHole.new end end @@ -32,20 +38,11 @@ def self.load_with_missing_constants(str_or_io) self.load(str_or_io) rescue Exception => e case e.message - when /undefined class\/module ((?:\w+::)+)$/ - names = $1.split('::') - missing = names.pop - base = names.inject(Object) { |m, n| m.const_get(n) } - base.const_set(missing, Module.new) - - if original_pos - str_or_io.seek(original_pos) - end - retry - when /undefined class\/module ((?:\w+::)+)(\w+)$/ - mod, klass = $1, $2 - full_name = "#{mod}#{klass}" - mod = mod.split('::').inject(Object) { |m, n| m.const_get(n) } + when /undefined class\/module ((?:\w+)(?:::\w+)*)(?:::)?$/ + full_name = $1 + path = $1.split('::') + *path, klass = *path + mod = path.inject(Object) { |m, n| m.const_get(n) } blackhole = Class.new(BlackHole) do @name = full_name diff --git a/test/test_marshal.rb b/test/test_marshal.rb new file mode 100644 index 0000000..97b110e --- /dev/null +++ b/test/test_marshal.rb @@ -0,0 +1,47 @@ +require 'utilrb/test' +require 'utilrb/marshal' + +module MarshalLoadWithMissingConstantsEnv +end + +describe Marshal do + describe "#load_with_missing_constants" do + after do + Object.const_set :MarshalLoadWithMissingConstantsEnv, Module.new + end + + it "resolves existing constants as expected" do + MarshalLoadWithMissingConstantsEnv.const_set 'Test', (klass = Class.new) + dumped = Marshal.dump(klass.new) + obj = Marshal.load_with_missing_constants(dumped) + assert_kind_of klass, obj + end + + it "creates missing classes as needed" do + MarshalLoadWithMissingConstantsEnv.const_set 'Test', (klass = Class.new) + dumped = Marshal.dump(klass.new) + Object.const_set :MarshalLoadWithMissingConstantsEnv, Module.new + obj = Marshal.load_with_missing_constants(dumped) + assert_same obj.class, MarshalLoadWithMissingConstantsEnv::Test + assert_kind_of Marshal::BlackHole, obj + assert_equal "MarshalLoadWithMissingConstantsEnv::Test", obj.class.name + end + + it "resolves missing namespaces recursively" do + MarshalLoadWithMissingConstantsEnv.const_set 'Test', (namespace = Module.new) + MarshalLoadWithMissingConstantsEnv::Test.const_set 'Test', (klass = Class.new) + dumped = Marshal.dump(klass.new) + Object.const_set :MarshalLoadWithMissingConstantsEnv, Module.new + obj = Marshal.load_with_missing_constants(dumped) + + blackhole_namespace = MarshalLoadWithMissingConstantsEnv::Test + assert(blackhole_namespace < Marshal::BlackHole) + assert_equal "MarshalLoadWithMissingConstantsEnv::Test", blackhole_namespace.name + assert_same obj.class, blackhole_namespace::Test + + assert_kind_of Marshal::BlackHole, obj + assert_equal "MarshalLoadWithMissingConstantsEnv::Test::Test", obj.class.name + end + end +end + From a224b5415f8bb6b396a7edcd613a647db9b9d583 Mon Sep 17 00:00:00 2001 From: Sylvain Joyeux Date: Mon, 18 Apr 2016 11:04:31 -0300 Subject: [PATCH 3/8] test: remove usage of simplecov deprecated APIs --- lib/utilrb/test.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/utilrb/test.rb b/lib/utilrb/test.rb index 0785def..64f580a 100644 --- a/lib/utilrb/test.rb +++ b/lib/utilrb/test.rb @@ -4,10 +4,10 @@ begin require 'simplecov' require 'coveralls' - SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[ - SimpleCov::Formatter::HTMLFormatter, - Coveralls::SimpleCov::Formatter - ] + SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new( + [SimpleCov::Formatter::HTMLFormatter, + Coveralls::SimpleCov::Formatter] + ) SimpleCov.start do add_filter "/test/" end From 650c6f12bb1d06a7c325d9af9ca90aa85b930004 Mon Sep 17 00:00:00 2001 From: Sylvain Joyeux Date: Mon, 18 Apr 2016 10:14:33 -0300 Subject: [PATCH 4/8] pkgconfig: make each_package without block return an enumerator --- lib/utilrb/pkgconfig.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/utilrb/pkgconfig.rb b/lib/utilrb/pkgconfig.rb index d87e3f0..649cae2 100644 --- a/lib/utilrb/pkgconfig.rb +++ b/lib/utilrb/pkgconfig.rb @@ -452,6 +452,8 @@ def self.has_package?(name) # Yields the package names of available packages. If +regex+ is given, # lists only the names that match the regular expression. def self.each_package(regex = nil) + return enum_for(__method__) if !block_given? + seen = Set.new each_pkgconfig_directory do |dir| Dir.glob(File.join(dir, '*.pc')) do |file| From feb1d948d33ed056502d14de8f6b430b3d395f3b Mon Sep 17 00:00:00 2001 From: Sylvain Joyeux Date: Mon, 18 Apr 2016 10:25:29 -0300 Subject: [PATCH 5/8] pkgconfig: remove cache The cache was a workaround for the badness of the way orogen was loading packages. Now that this is fixed, it does more harm than good since the pkgconfig objects end in the long-lived objects (something that can end up being up to 50k objects ...) --- lib/utilrb/pkgconfig.rb | 25 ++++++------------------- test/test_pkgconfig.rb | 1 - 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/lib/utilrb/pkgconfig.rb b/lib/utilrb/pkgconfig.rb index 649cae2..ce6ac63 100644 --- a/lib/utilrb/pkgconfig.rb +++ b/lib/utilrb/pkgconfig.rb @@ -41,15 +41,6 @@ class PkgConfig VAR_NAME_RX = /\w+/ FIELD_NAME_RX = /[\w\.\-]+/ - class << self - attr_reader :loaded_packages - - def clear_cache - loaded_packages.clear - end - end - @loaded_packages = Hash.new - def self.load(path, preset_variables) pkg_name = File.basename(path, ".pc") pkg = Class.instance_method(:new).bind(PkgConfig).call(pkg_name) @@ -60,17 +51,13 @@ def self.load(path, preset_variables) # Returns the pkg-config object that matches the given name, and # optionally a version string def self.get(name, version_spec = nil, preset_variables = Hash.new) - if !(candidates = loaded_packages[name]) - paths = find_all_package_files(name) - if paths.empty? - raise NotFound.new(name), "cannot find the pkg-config specification for #{name}" - end + paths = find_all_package_files(name) + if paths.empty? + raise NotFound.new(name), "cannot find the pkg-config specification for #{name}" + end - candidates = Array.new - paths.each do |p| - candidates << PkgConfig.load(p, preset_variables) - end - loaded_packages[name] = candidates + candidates = paths.map do |p| + PkgConfig.load(p, preset_variables) end # Now try to find a matching spec diff --git a/test/test_pkgconfig.rb b/test/test_pkgconfig.rb index 594b03f..dbf43d4 100644 --- a/test/test_pkgconfig.rb +++ b/test/test_pkgconfig.rb @@ -9,7 +9,6 @@ def setup end def teardown ENV['PKG_CONFIG_PATH'] = @old_pkg_config_path - PkgConfig.clear_cache end PkgConfig = Utilrb::PkgConfig From cdf69911ca87c83e8f0319695ed1853002aaba55 Mon Sep 17 00:00:00 2001 From: Sylvain Joyeux Date: Mon, 18 Apr 2016 13:16:19 -0300 Subject: [PATCH 6/8] pkgconfig: refactor #load into smaller parts --- lib/utilrb/pkgconfig.rb | 129 ++++++++++++++++++++++++++++------------ test/test_pkgconfig.rb | 2 - 2 files changed, 91 insertions(+), 40 deletions(-) diff --git a/lib/utilrb/pkgconfig.rb b/lib/utilrb/pkgconfig.rb index ce6ac63..c2a29b4 100644 --- a/lib/utilrb/pkgconfig.rb +++ b/lib/utilrb/pkgconfig.rb @@ -48,24 +48,36 @@ def self.load(path, preset_variables) pkg end + def self.load_minimal(path, preset_variables) + pkg_name = File.basename(path, ".pc") + pkg = Class.instance_method(:new).bind(PkgConfig).call(pkg_name) + pkg.load_minimal(path, preset_variables) + pkg + end + # Returns the pkg-config object that matches the given name, and # optionally a version string - def self.get(name, version_spec = nil, preset_variables = Hash.new) + def self.get(name, version_spec = nil, preset_variables = Hash.new, minimal: false) paths = find_all_package_files(name) if paths.empty? raise NotFound.new(name), "cannot find the pkg-config specification for #{name}" end candidates = paths.map do |p| - PkgConfig.load(p, preset_variables) + PkgConfig.load_minimal(p, preset_variables) end # Now try to find a matching spec - if version_match = find_matching_version(candidates, version_spec) - version_match + if match = find_matching_version(candidates, version_spec) + match else raise NotFound, "found #{candidates.size} packages for #{name}, but none match the version specification #{version_spec}" end + + if !minimal + match.load_fields + end + match end # Finds the provided package and optional version and returns its @@ -127,7 +139,6 @@ def initialize(name); @name = name end end - attr_reader :file attr_reader :path # The module name @@ -138,6 +149,8 @@ def initialize(name); @name = name end # The module version, as an array of integers attr_reader :version + attr_reader :raw_fields + # Information extracted from the file attr_reader :variables attr_reader :fields @@ -156,7 +169,7 @@ def initialize(name) # +current+ is a string that describes what we are expanding. It is used # to detect recursion in expansion of variables, and to give meaningful # errors to the user - def expand_variables(value, variables, current) + def perform_substitution(value, variables, current) value = value.gsub(/\$\{(\w+)\}/) do |rx| expand_name = $1 if expand_name == current @@ -198,16 +211,13 @@ def self.parse_dependencies(string) SHELL_VARS = %w{Cflags Libs Libs.private} - # Loads the information contained in +path+ - def load(path, preset_variables = Hash.new) - @path = path - @file = File.readlines(path).map(&:strip) - - raw_variables = preset_variables.dup - raw_fields = Hash.new - + # Parse a pkg-config field and extracts the raw definition of variables + # and fields + # + # @return [(Hash,Hash)] the set of variables and the set of fields + def parse(path) running_line = nil - @file = file.map do |line| + file = File.readlines(path).map do |line| line = line.gsub(/\s*#.*$/, '') line = line.strip next if line.empty? @@ -227,6 +237,7 @@ def load(path, preset_variables = Hash.new) end.compact + raw_variables, raw_fields = Hash.new, Hash.new file.each do |line| case line when /^(#{VAR_NAME_RX})\s*=(.*)/ @@ -237,44 +248,78 @@ def load(path, preset_variables = Hash.new) raise NotImplementedError, "#{path}: cannot parse pkg-config line #{line.inspect}" end end + return raw_variables, raw_fields + end + + def expand_variables(raw_variables) + raw_variables = raw_variables.dup + variables = Hash.new # Resolve the variables while variables.size != raw_variables.size raw_variables.each do |name, value| - value = expand_variables(value, raw_variables, name) + value = perform_substitution(value, raw_variables, name) raw_variables[name] = value if value !~ /\$\{#{VAR_NAME_RX}\}/ variables[name] = value end end end - - # Shell-split the fields, and expand the variables in them - raw_fields.each do |name, value| - if SHELL_VARS.include?(name) - value = Shellwords.shellsplit(value) - resolved = Array.new - while !value.empty? - value = value.flat_map do |v| - expanded = expand_variables(v, variables, name) - if expanded == v - resolved << v - nil - else - Shellwords.shellsplit(expanded) - end - end.compact - end - fields[name] = resolved - else - fields[name] = expand_variables(value, variables, name) + variables + end + + def expand_field(name, field) + if SHELL_VARS.include?(name) + value = Shellwords.shellsplit(field) + resolved = Array.new + while !value.empty? + value = value.flat_map do |v| + expanded = perform_substitution(v, variables, name) + if expanded == v + resolved << v + nil + else + Shellwords.shellsplit(expanded) + end + end.compact end + resolved + else + perform_substitution(field, variables, name) + end + end + + def load_variables(path, preset_variables = Hash.new) + raw_variables, raw_fields = parse(path) + raw_variables = preset_variables.merge(raw_variables) + expand_variables(raw_variables) + end + + def load_minimal(path, preset_variables = Hash.new) + raw_variables, raw_fields = parse(path) + raw_variables = preset_variables.merge(raw_variables) + + @variables = expand_variables(raw_variables) + if raw_fields['Version'] + @raw_version = expand_field('Version', raw_fields['Version']) + else + @raw_version = '' + end + @version = raw_version.split('.').map { |v| Integer(v) if v =~ /^\d+$/ }.compact + # To be used in the call to #load + @raw_fields = raw_fields + @path = path + end + + def load_fields + fields = Hash.new + @raw_fields.each do |name, value| + fields[name] = expand_field(name, value) end + @fields = fields # Initialize the main flags - @raw_version = (fields['Version'] || '') - @version = raw_version.split('.').map { |v| Integer(v) if v =~ /^\d+$/ }.compact @description = (fields['Description'] || '') # Get the requires/conflicts @@ -313,6 +358,14 @@ def load(path, preset_variables = Hash.new) end end + # Loads the information contained in +path+ + def load(path, preset_variables = Hash.new) + if !@raw_fields + load_minimal(path, preset_variables) + end + load_fields + end + def self.define_pkgconfig_action(action) # :nodoc: class_eval <<-EOD, __FILE__, __LINE__+1 def pkgconfig_#{action.gsub(/-/, '_')}(static = false) diff --git a/test/test_pkgconfig.rb b/test/test_pkgconfig.rb index dbf43d4..e409c01 100644 --- a/test/test_pkgconfig.rb +++ b/test/test_pkgconfig.rb @@ -89,8 +89,6 @@ def test_comparison_with_cpkgconfig puts "#{name} #{action_name}" puts " pure ruby: #{pure_ruby.inspect}" puts " cpkgconfig: #{cpkgconfig.inspect}" - puts "contents:" - puts pkg.file.join("\n") end end end From 6de5188f4061032a31ff3346ce84b924aee16f71 Mon Sep 17 00:00:00 2001 From: Sylvain Joyeux Date: Mon, 2 May 2016 09:56:46 -0300 Subject: [PATCH 7/8] pkgconfig: re-add .clear_cache for backward compatibility --- lib/utilrb/pkgconfig.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/utilrb/pkgconfig.rb b/lib/utilrb/pkgconfig.rb index c2a29b4..20453eb 100644 --- a/lib/utilrb/pkgconfig.rb +++ b/lib/utilrb/pkgconfig.rb @@ -55,6 +55,11 @@ def self.load_minimal(path, preset_variables) pkg end + # @deprecated {PkgConfig} does not cache the packages anymore, so no + # need to call this method + def self.clear_cache + end + # Returns the pkg-config object that matches the given name, and # optionally a version string def self.get(name, version_spec = nil, preset_variables = Hash.new, minimal: false) From 153e0a614ac046129cfba491b2905ebffb6c30cf Mon Sep 17 00:00:00 2001 From: Sylvain Joyeux Date: Mon, 2 May 2016 09:26:46 -0300 Subject: [PATCH 8/8] pkgconfig: allow to use a non-default path on all search methods --- lib/utilrb/pkgconfig.rb | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/utilrb/pkgconfig.rb b/lib/utilrb/pkgconfig.rb index 20453eb..94f6e1b 100644 --- a/lib/utilrb/pkgconfig.rb +++ b/lib/utilrb/pkgconfig.rb @@ -62,8 +62,8 @@ def self.clear_cache # Returns the pkg-config object that matches the given name, and # optionally a version string - def self.get(name, version_spec = nil, preset_variables = Hash.new, minimal: false) - paths = find_all_package_files(name) + def self.get(name, version_spec = nil, preset_variables = Hash.new, minimal: false, pkg_config_path: self.pkg_config_path) + paths = find_all_package_files(name, pkg_config_path: pkg_config_path) if paths.empty? raise NotFound.new(name), "cannot find the pkg-config specification for #{name}" end @@ -460,17 +460,22 @@ def method_missing(varname, *args, &proc) # :nodoc: end end - def self.each_pkgconfig_directory(&block) - if path = ENV['PKG_CONFIG_PATH'] - path.split(':').each(&block) + def self.pkg_config_path + ENV['PKG_CONFIG_PATH'] + end + + def self.each_pkgconfig_directory(pkg_config_path: self.pkg_config_path, &block) + return enum_for(__method__) if !block_given? + if pkg_config_path + pkg_config_path.split(':').each(&block) end default_search_path.each(&block) end # Returns true if there is a package with this name - def self.find_all_package_files(name) + def self.find_all_package_files(name, pkg_config_path: self.pkg_config_path) result = [] - each_pkgconfig_directory do |dir| + each_pkgconfig_directory(pkg_config_path: pkg_config_path) do |dir| path = File.join(dir, "#{name}.pc") if File.exist?(path) result << path @@ -479,9 +484,9 @@ def self.find_all_package_files(name) result end - def self.available_package_names + def self.available_package_names(pkg_config_path: self.pkg_config_path) result = [] - each_pkgconfig_directory do |dir| + each_pkgconfig_directory(pkg_config_path: pkg_config_path) do |dir| Dir.glob(File.join(dir, "*.pc")) do |path| result << File.basename(path, ".pc") end @@ -490,17 +495,17 @@ def self.available_package_names end # Returns true if there is a package with this name - def self.has_package?(name) - !find_all_package_files(name).empty? + def self.has_package?(name, pkg_config_path: self.pkg_config_path) + !find_all_package_files(name, pkg_config_path: pkg_config_path).empty? end # Yields the package names of available packages. If +regex+ is given, # lists only the names that match the regular expression. - def self.each_package(regex = nil) + def self.each_package(regex = nil, pkg_config_path: self.pkg_config_path) return enum_for(__method__) if !block_given? seen = Set.new - each_pkgconfig_directory do |dir| + each_pkgconfig_directory(pkg_config_path: pkg_config_path) do |dir| Dir.glob(File.join(dir, '*.pc')) do |file| pkg_name = File.basename(file, ".pc") next if seen.include?(pkg_name)