diff --git a/lib/gtfs_df/feed.rb b/lib/gtfs_df/feed.rb index 7df438b..a793bdd 100644 --- a/lib/gtfs_df/feed.rb +++ b/lib/gtfs_df/feed.rb @@ -36,7 +36,7 @@ class Feed booking_rules ].freeze - attr_reader(*GTFS_FILES) + attr_reader(*GTFS_FILES, :graph) # Initialize with a hash of DataFrames REQUIRED_GTFS_FILES = %w[agency stops routes trips stop_times].freeze @@ -53,6 +53,8 @@ def initialize(data = {}) end.join(", ")}" end + @graph = GtfsDf::Graph.build + GTFS_FILES.each do |file| df = data[file] schema_class_name = file.split("_").map(&:capitalize).join @@ -68,81 +70,36 @@ def initialize(data = {}) end end - # Load from a directory of GTFS CSV files - def self.load_from_dir(dir) - data = {} - GTFS_FILES.each do |file| - path = File.join(dir, "#{file}.txt") - next unless File.exist?(path) - - schema_class_name = file.split("_").map(&:capitalize).join - - data[file] = GtfsDf::Schema.const_get(schema_class_name).new(path) - end - new(data) - end - # Filter the feed using a view hash # Example view: { 'routes' => { 'route_id' => '123' }, 'trips' => { 'service_id' => 'A' } } def filter(view) filtered = {} - graph = GtfsDf::Graph.build - # Step 1: Apply view filters + GTFS_FILES.each do |file| df = send(file) next unless df - filters = view[file] - if filters && !filters.empty? - filters.each do |col, val| - df = if val.is_a?(Array) - df.filter(Polars.col(col).is_in(val)) - elsif val.respond_to?(:call) - df.filter(val.call(Polars.col(col))) - else - df.filter(Polars.col(col).eq(val)) - end - end - end filtered[file] = df end - # Step 2: Cascade filters following the directed edges - # An edge from parent->child means: filter child based on valid parent IDs - changed = true - while changed - changed = false - GTFS_FILES.each do |parent_file| - parent_df = filtered[parent_file] - next unless parent_df - - # For each outgoing edge from parent_file to child_file - graph.adj[parent_file]&.each do |child_file, attrs| - child_df = filtered[child_file] - next unless child_df && child_df.height > 0 - - attrs[:dependencies].each do |dep| - parent_col = dep[parent_file] - child_col = dep[child_file] - - next unless parent_col && child_col && - parent_df.columns.include?(parent_col) && child_df.columns.include?(child_col) - - # Get valid values from parent - valid_values = parent_df[parent_col].to_a.uniq.compact - - # Filter child to only include rows that reference valid parent values - before = child_df.height - child_df = child_df.filter(Polars.col(child_col).is_in(valid_values)) - - if child_df.height < before - filtered[child_file] = child_df - changed = true - end - end - end + + # Trips are the atomic unit of GTFS, we will generate a new view + # based on the set of trips that would be included for each invidual filter + # and cascade changes from this view in order to retain referential integrity + trip_ids = nil + + view.each do |file, filters| + new_filtered = filter!(file, filters, filtered.dup) + trip_ids = if trip_ids.nil? + new_filtered["trips"]["trip_id"] + else + trip_ids & new_filtered["trips"]["trip_id"] end end + if trip_ids + filtered = filter!("trips", {"trip_id" => trip_ids.to_a}, filtered) + end + # Remove files that are empty, but keep required files even if empty filtered.delete_if do |file, df| is_required_file = REQUIRED_GTFS_FILES.include?(file) || @@ -153,5 +110,60 @@ def filter(view) end self.class.new(filtered) end + + private + + def filter!(file, filters, filtered) + unless filters.empty? + df = filtered[file] + + filters.each do |col, val| + df = if val.is_a?(Array) + df.filter(Polars.col(col).is_in(val)) + elsif val.respond_to?(:call) + df.filter(val.call(Polars.col(col))) + else + df.filter(Polars.col(col).eq(val)) + end + end + + filtered[file] = df + + prune!(file, filtered) + end + + filtered + end + + def prune!(root, filtered) + graph.each_bfs_edge(root) do |parent_file, child_file| + parent_df = filtered[parent_file] + next unless parent_df + + child_df = filtered[child_file] + next unless child_df && child_df.height > 0 + + attrs = graph.get_edge_data(parent_file, child_file) + + attrs[:dependencies].each do |dep| + parent_col = dep[parent_file] + child_col = dep[child_file] + + next unless parent_col && child_col && + parent_df.columns.include?(parent_col) && child_df.columns.include?(child_col) + + # Get valid values from parent + valid_values = parent_df[parent_col].to_a.uniq.compact + + # Filter child to only include rows that reference valid parent values + before = child_df.height + child_df = child_df.filter(Polars.col(child_col).is_in(valid_values)) + + if child_df.height < before + filtered[child_file] = child_df + end + end + end + end end end diff --git a/lib/gtfs_df/graph.rb b/lib/gtfs_df/graph.rb index 9a5df37..98669a5 100644 --- a/lib/gtfs_df/graph.rb +++ b/lib/gtfs_df/graph.rb @@ -4,7 +4,7 @@ module GtfsDf class Graph # Returns a directed graph of GTFS file dependencies def self.build - g = NetworkX::DiGraph.new + g = NetworkX::Graph.new # Nodes: GTFS files files = %w[ agency routes trips stop_times stops calendar calendar_dates shapes transfers frequencies fare_attributes fare_rules @@ -12,7 +12,7 @@ def self.build ] files.each { |f| g.add_node(f) } - # Edges: dependencies + # TODO: Add fare_rules -> stops + test edges = [ ["agency", "routes", {dependencies: [ {"agency" => "agency_id", "routes" => "agency_id"} @@ -116,9 +116,6 @@ def self.build ["booking_rules", "stop_times", {dependencies: [ {"booking_rules" => "booking_rule_id", "stop_times" => "pickup_booking_rule_id"}, {"booking_rules" => "booking_rule_id", "stop_times" => "drop_off_booking_rule_id"} - ]}], - ["stops", "booking_rules", {dependencies: [ - {"stops" => "stop_id", "booking_rules" => "stop_id"} ]}] ] diff --git a/spec/gtfs_df/feed_spec.rb b/spec/gtfs_df/feed_spec.rb index 28e806e..9b63d2a 100644 --- a/spec/gtfs_df/feed_spec.rb +++ b/spec/gtfs_df/feed_spec.rb @@ -59,29 +59,154 @@ end describe ".filter" do - it "filters DataFrames using the view hash" do - expect(feed.routes["route_id"].to_a).to eq(%w[1 2]) - expect(feed.trips["service_id"].to_a).to eq(%w[A B]) + describe "basic functionality" do + it "filters DataFrames using the view hash" do + expect(feed.routes["route_id"].to_a).to eq(%w[1 2]) + expect(feed.trips["service_id"].to_a).to eq(%w[A B]) - view = {"routes" => {"route_id" => "1"}} - filtered = feed.filter(view) + view = {"routes" => {"route_id" => "1"}} + filtered = feed.filter(view) - expect(filtered.routes["route_id"].to_a).to eq(["1"]) - expect(filtered.trips["service_id"].to_a).to eq(%w[A]) + expect(filtered.routes["route_id"].to_a).to eq(["1"]) + expect(filtered.trips["service_id"].to_a).to eq(%w[A]) + end + + it "filters DataFrames using multiple values" do + view = {"routes" => {"route_id" => %w[1 2]}} + filtered = feed.filter(view) + expect(filtered.routes["route_id"].to_a).to eq(%w[1 2]) + expect(filtered.trips["route_id"].to_a).to eq(%w[1 2]) + end + + it "filters DataFrames using a custom predicate" do + view = {"routes" => {"route_id" => ->(col) { col.str.starts_with("1") }}} + filtered = feed.filter(view) + expect(filtered.routes["route_id"].to_a).to eq(["1"]) + expect(filtered.trips["route_id"].to_a).to eq(["1"]) + end end - it "filters DataFrames using multiple values" do - view = {"routes" => {"route_id" => %w[1 2]}} - filtered = feed.filter(view) - expect(filtered.routes["route_id"].to_a).to eq(%w[1 2]) - expect(filtered.trips["route_id"].to_a).to eq(%w[1 2]) + describe "filtering through the graph" do + let(:agency_df) do + Polars::DataFrame.new({"agency_id" => ["A", "B"], + "agency_name" => ["Test A", "Test B"], + "agency_url" => ["http://agencyA", "http://agencyB"], + "agency_timezone" => ["America/Chicago", "America/Chicago"]}) + end + let(:routes_df) do + Polars::DataFrame.new({"route_id" => %w[1 2], + "agency_id" => %w[A B], + "route_short_name" => %w[A B]}) + end + let(:stops_df) do + Polars::DataFrame.new({"stop_id" => %w[S1 S2 S3 S4], + "stop_name" => %w[Stop1 Stop2 Stop3 Stop4], + "stop_lat" => %w[0 1 2 3], + "stop_lon" => %w[0 1 2 3]}) + end + # Trip 1 visits stops 1,2,3 + # Trip 2 visits stops 2,4 + let(:stop_times_df) do + Polars::DataFrame.new({"trip_id" => %w[t1 t1 t1 t2 t2], + "stop_id" => %w[S1 S2 S3 S2 S4], + "stop_sequence" => [1, 2, 3, 1, 2]}) + end + + it "filtering from trips cascades" do + # We consider trips the "atomic unit" of GTFS + view = {"trips" => {"trip_id" => %w[t1]}} + filtered = feed.filter(view) + + # When trips are filtered, we expect the filters to propagate to stop_times + expect(filtered.stop_times["stop_id"].to_a).to eq(%w[S1 S2 S3]) + + # Remove unreferenced objects + expect(filtered.stops["stop_id"].to_a).to eq(%w[S1 S2 S3]) + expect(filtered.routes["route_id"].to_a).to eq(%w[1]) + expect(filtered.agency["agency_id"].to_a).to eq(%w[A]) + expect(filtered.calendar["service_id"].to_a).to eq(%w[A]) + end + + it "filtering from stop cascades" do + # We consider trips the "atomic unit" of GTFS + # This means if we filter to a stop, we should essentially filter to all trips + # which visit that stop and retain referential integrity for those trips + # This stop is found only in T1 + view = {"stops" => {"stop_id" => %w[S1]}} + filtered = feed.filter(view) + + # Retain all stop times for T1 but not T2 + expect(filtered.stop_times["stop_id"].to_a).to eq(%w[S1 S2 S3]) + + # Retain all stops referenced by T1 T1 + expect(filtered.stops["stop_id"].to_a).to eq(%w[S1 S2 S3]) + expect(filtered.routes["route_id"].to_a).to eq(%w[1]) + expect(filtered.agency["agency_id"].to_a).to eq(%w[A]) + expect(filtered.calendar["service_id"].to_a).to eq(%w[A]) + end + + it "filtering from agency cascades" do + # Filtering by agency should cascade to routes -> trips -> etc. + view = {"agency" => {"agency_id" => %w[A]}} + filtered = feed.filter(view) + + expect(filtered.agency["agency_id"].to_a).to eq(%w[A]) + + expect(filtered.routes["route_id"].to_a).to eq(%w[1]) + expect(filtered.trips["trip_id"].to_a).to eq(%w[t1]) + expect(filtered.stop_times["stop_id"].to_a).to eq(%w[S1 S2 S3]) + + # Remove unreferenced objects + expect(filtered.stops["stop_id"].to_a).to eq(%w[S1 S2 S3]) + expect(filtered.calendar["service_id"].to_a).to eq(%w[A]) + end + + it "filtering from calendar cascades" do + # Filtering by agency should cascade to routes -> trips -> etc. + view = {"calendar" => {"service_id" => %w[A]}} + filtered = feed.filter(view) + + expect(filtered.calendar["service_id"].to_a).to eq(%w[A]) + + # Only keep trips and stop times for this service + expect(filtered.trips["trip_id"].to_a).to eq(%w[t1]) + expect(filtered.stop_times["stop_id"].to_a).to eq(%w[S1 S2 S3]) + + # Remove unreferenced objects + expect(filtered.stops["stop_id"].to_a).to eq(%w[S1 S2 S3]) + expect(filtered.routes["route_id"].to_a).to eq(%w[1]) + expect(filtered.agency["agency_id"].to_a).to eq(%w[A]) + end end - it "filters DataFrames using a custom predicate" do - view = {"routes" => {"route_id" => ->(col) { col.str.starts_with("1") }}} - filtered = feed.filter(view) - expect(filtered.routes["route_id"].to_a).to eq(["1"]) - expect(filtered.trips["route_id"].to_a).to eq(["1"]) + describe "edge cases" do + let(:feed) do + described_class.new({"agency" => agency_df, + "stops" => stops_df, + "routes" => routes_df, + "trips" => trips_df, + "stop_times" => stop_times_df, + "calendar" => calendar_df}) + end + + it "returns empty feed when filter matches nothing" do + view = {"routes" => {"route_id" => "NONEXISTENT"}} + filtered = feed.filter(view) + expect(filtered.routes.height).to eq(0) + # Trips and their stop_times should cascade and be empty since no valid routes exist + expect(filtered.trips.height).to eq(0) + expect(filtered.stop_times.height).to eq(0) + # No trips reference this calendar so deletion is cascaded + expect(filtered.calendar.height).to eq(0) + # No trips reference this agency so deletion is cascaded + expect(filtered.agency.height).to eq(0) + end + + it "handles filtering with empty array" do + view = {"routes" => {"route_id" => []}} + filtered = feed.filter(view) + expect(filtered.routes.height).to eq(0) + end end end @@ -177,37 +302,6 @@ end end - describe "filtering edge cases" do - let(:feed) do - described_class.new({"agency" => agency_df, - "stops" => stops_df, - "routes" => routes_df, - "trips" => trips_df, - "stop_times" => stop_times_df, - "calendar" => calendar_df}) - end - - it "returns empty feed when filter matches nothing" do - view = {"routes" => {"route_id" => "NONEXISTENT"}} - filtered = feed.filter(view) - expect(filtered.routes.height).to eq(0) - # Trips and their stop_times should cascade and be empty since no valid routes exist - expect(filtered.trips.height).to eq(0) - expect(filtered.stop_times.height).to eq(0) - # No trips reference this calendar so deletion is cascaded - expect(filtered.calendar.height).to eq(0) - - # Agency and calendar are not dependencies and should be untouched - expect(filtered.agency.height).to eq(1) - end - - it "handles filtering with empty array" do - view = {"routes" => {"route_id" => []}} - filtered = feed.filter(view) - expect(filtered.routes.height).to eq(0) - end - end - describe "GTFS extension file support" do let(:minimal_feed_data) do { @@ -323,7 +417,8 @@ "service_id" => ["S1"]}), "stop_times" => Polars::DataFrame.new({"trip_id" => ["T1"], "stop_id" => ["S1"], - "stop_sequence" => [1]}), + "stop_sequence" => [1], + "pickup_booking_rule_id" => %w[BR1]}), "calendar" => Polars::DataFrame.new({"service_id" => ["S1"], "monday" => ["1"], "tuesday" => ["1"], @@ -347,7 +442,6 @@ "location_group_stops" => Polars::DataFrame.new({"location_group_id" => %w[LG1 LG2], "stop_id" => %w[S1 S2]}), "booking_rules" => Polars::DataFrame.new({"booking_rule_id" => %w[BR1 BR2], - "stop_id" => %w[S1 S2], "booking_method" => %w[phone web], "info_url" => ["http://a", "http://b"]})}) filtered = feed.filter({"stops" => {stop_id: ["S1"]}}) @@ -355,7 +449,7 @@ expect(filtered.areas["area_id"].to_a).to eq(["A1"]) expect(filtered.location_groups["location_group_id"].to_a).to eq(["LG1"]) expect(filtered.location_group_stops["stop_id"].to_a).to eq(["S1"]) - expect(filtered.booking_rules["stop_id"].to_a).to eq(["S1"]) + expect(filtered.booking_rules["booking_rule_id"].to_a).to eq(["BR1"]) end end end diff --git a/spec/gtfs_df/graph_spec.rb b/spec/gtfs_df/graph_spec.rb index 35f0ba5..a18c4f6 100644 --- a/spec/gtfs_df/graph_spec.rb +++ b/spec/gtfs_df/graph_spec.rb @@ -5,8 +5,7 @@ let(:graph) { described_class.build } it "includes all GTFS files as nodes" do - node_names = graph.nodes.map { |n| n[0] } - expect(node_names.sort).to include(*%w[agency + expect(graph.nodes.sort).to include(*%w[agency routes trips stop_times @@ -91,11 +90,12 @@ "stops")[:dependencies]).to include({"location_group_stops" => "stop_id", "stops" => "stop_id"}) - # Example: stops -> booking_rules - expect(graph.has_edge?("stops", "booking_rules")).to be true - expect(graph.get_edge_data("stops", - "booking_rules")[:dependencies]).to include({"stops" => "stop_id", - "booking_rules" => "stop_id"}) + # Example: stop_times -> booking_rules + expect(graph.has_edge?("booking_rules", "stop_times")).to be true + expect(graph.get_edge_data("booking_rules", + "stop_times")[:dependencies]).to include( + {"booking_rules" => "booking_rule_id", "stop_times" => "pickup_booking_rule_id"} + ) end end end diff --git a/spec/gtfs_df/writer_spec.rb b/spec/gtfs_df/writer_spec.rb index adedce6..b6cf699 100644 --- a/spec/gtfs_df/writer_spec.rb +++ b/spec/gtfs_df/writer_spec.rb @@ -42,7 +42,6 @@ # Check that only the filtered route_id is present lines = routes_txt.lines.map(&:strip) - lines.first data_lines = lines.drop(1) expect(data_lines.size).to eq(1) expect(data_lines.first).to include(route_ids.first)