feat(datasource-pylon): PylonIssue read-only collection with cursor pagination

.rubocop.yml

@@ -258,6 +258,7 @@ Naming/PredicatePrefix:
 Metrics/ParameterLists:
   Exclude:
     - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb'
+    - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/base_collection.rb'
     - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb'
     - 'packages/forest_admin_agent/lib/forest_admin_agent/routes/query_handler.rb'
     - 'packages/forest_admin_agent/lib/forest_admin_agent/services/smart_action_checker.rb'

packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/client.rb

@@ -1,5 +1,11 @@
 module ForestAdminDatasourcePylon
   class Client
+    MAX_SEARCH_LIMIT = 1000
+
+    # `next_cursor` is nil as soon as Pylon stops advertising a next page, so
+    # callers never have to know how the absence is spelled on the wire.
+    SearchPage = Struct.new(:records, :next_cursor, keyword_init: true)
+
     def initialize(configuration)
       @configuration = configuration
     end
@@ -10,8 +16,46 @@ def me
       must_succeed('me') { extract_data(connection.get('me').body) }
     end
 
+    # POST /issues/search accepts an empty body and then returns the most recent
+    # issues, ordered by `created_at` descending.
+    def search_issues(limit:, cursor: nil, filter: nil, search_text: nil)
+      body = { 'limit' => clamp_limit(limit) }
+      body['cursor']      = cursor unless blank?(cursor)
+      body['filter']      = filter unless filter.nil?
+      body['search_text'] = search_text unless blank?(search_text)
+
+      must_succeed('issues/search') { to_search_page(connection.post('issues/search', body).body) }
+    end
+
+    # Accepts either the UUID or the issue number. The id comes from
+    # operator-supplied filter values, so it is escaped before joining the path.
+    def fetch_issue(id)
+      path = "issues/#{Faraday::Utils.escape(id)}"
+      must_succeed(path) { extract_data(connection.get(path).body) }
+    end
+
     private
 
+    def clamp_limit(limit)
+      value = limit.to_i
+      return 1 if value < 1
+
+      [value, MAX_SEARCH_LIMIT].min
+    end
+
+    # Pylon only includes the `pagination` block when a next page exists, so an
+    # absent block, `has_next_page: false` and an empty cursor all mean "done".
+    def to_search_page(body)
+      pagination = body.is_a?(Hash) ? body['pagination'] : nil
+      cursor = pagination.is_a?(Hash) && pagination['has_next_page'] ? pagination['cursor'] : nil
+
+      SearchPage.new(records: Array(extract_data(body)), next_cursor: blank?(cursor) ? nil : cursor)
+    end
+
+    def blank?(value)
+      value.nil? || value.to_s.empty?
+    end
+
     # Pylon wraps payloads in { "data": ..., "pagination": ..., "request_id": ... }.
     def extract_data(body)
       return nil if body.nil? || body == ''

packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/base_collection.rb

@@ -0,0 +1,83 @@
+module ForestAdminDatasourcePylon
+  module Collections
+    class BaseCollection < ForestAdminDatasourceToolkit::Collection
+      ColumnSchema = ForestAdminDatasourceToolkit::Schema::ColumnSchema
+      Operators    = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators
+      Leaf         = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf
+
+      STRING_OPS = [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN,
+                    Operators::PRESENT, Operators::BLANK].freeze
+      NUMBER_OPS = (STRING_OPS + [Operators::GREATER_THAN, Operators::LESS_THAN]).freeze
+      DATE_OPS   = [Operators::EQUAL, Operators::BEFORE, Operators::AFTER,
+                    Operators::PRESENT, Operators::BLANK].freeze
+
+      attr_reader :custom_fields
+
+      # Template method: subclasses implement `define_schema` and
+      # `define_relations` as hooks; ordering between them, custom-field
+      # registration, and the search/count flags is owned here so collisions
+      # are always evaluated against the final native schema.
+      def initialize(datasource, name, custom_fields: [], searchable: false, countable: false, native_driver: nil)
+        super(datasource, name, native_driver)
+        define_schema
+        define_relations
+        @custom_fields = add_custom_fields(custom_fields)
+        enable_search if searchable
+        enable_count if countable
+      end
+
+      protected
+
+      # Pylon has no `id` filter operator on /issues/search, so collections
+      # short-circuit primary-key lookups to /resource/{id}. Ids are UUID
+      # strings — unlike Zendesk, nothing has to be coerced to an integer.
+      def extract_id_lookup(node)
+        return nil unless node.is_a?(Leaf) && node.field == 'id'
+
+        return unless [Operators::EQUAL, Operators::IN].include?(node.operator)
+
+        Array(node.value).map(&:to_s).reject(&:empty?)
+      end
+
+      def project(record, projection)
+        return record if projection.nil?
+
+        wanted = Array(projection).map(&:to_s).reject { |p| p.include?(':') }
+        return record if wanted.empty?
+
+        wanted.to_h { |k| [k, record[k]] }
+      end
+
+      def translate_page(page)
+        return [0, Client::MAX_SEARCH_LIMIT] if page.nil?
+
+        limit = page.limit.to_i.positive? ? page.limit.to_i : Client::MAX_SEARCH_LIMIT
+        [page.offset.to_i.clamp(0, nil), limit]
+      end
+
+      # Adds custom fields, skipping any whose column name collides with a
+      # field already declared on the collection. Returns the subset actually
+      # added so callers can keep their serializer in sync with the schema.
+      def add_custom_fields(custom_fields)
+        custom_fields.reject do |cf|
+          column_name = cf[:column_name]
+          if schema[:fields].key?(column_name)
+            ForestAdminDatasourcePylon.logger.warn(
+              "[forest_admin_datasource_pylon] Custom field '#{column_name}' on collection " \
+              "'#{name}' conflicts with an existing field; skipping."
+            )
+            true
+          else
+            add_field(column_name, cf[:schema])
+            false
+          end
+        end
+      end
+
+      private
+
+      def define_schema    = raise(NotImplementedError, "#{self.class} did not implement define_schema")
+      def define_relations = raise(NotImplementedError, "#{self.class} did not implement define_relations")
+    end
+  end
+end

packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue.rb

@@ -0,0 +1,67 @@
+module ForestAdminDatasourcePylon
+  module Collections
+    class Issue < BaseCollection
+      include SchemaDefinition
+      include Serializer
+
+      def initialize(datasource, custom_fields: [])
+        super(datasource, 'PylonIssue', custom_fields: custom_fields)
+      end
+
+      def list(_caller, filter, projection)
+        fetch_records(filter).map { |issue| project(serialize(issue), projection) }
+      end
+
+      private
+
+      def fetch_records(filter)
+        ids = extract_id_lookup(filter&.condition_tree)
+        return page_window(fetch_by_ids(ids), filter) if ids
+
+        warn_ignored_filter(filter)
+        offset, limit = translate_page(filter&.page)
+        walker.walk(offset: offset, limit: limit) do |batch, cursor|
+          datasource.client.search_issues(limit: batch, cursor: cursor)
+        end
+      end
+
+      # Sliced after the lookup, not before, so ids that resolved to nothing
+      # (404) do not eat into the requested window.
+      def page_window(records, filter)
+        offset, limit = translate_page(filter&.page)
+        records[offset, limit] || []
+      end
+
+      # A record the operator can no longer reach — deleted, or outside the
+      # token's scope — reads as "no record" rather than as a failed page.
+      def fetch_by_ids(ids)
+        ids.filter_map do |id|
+          datasource.client.fetch_issue(id)
+        rescue APIError => e
+          raise unless e.status == 404
+
+          nil
+        end
+      end
+
+      # Condition-tree translation and free-text search land in a later story.
+      # Until then a filter the collection cannot honour is dropped, which would
+      # otherwise silently return unfiltered rows.
+      def warn_ignored_filter(filter)
+        ignored = []
+        ignored << 'condition tree' unless filter&.condition_tree.nil?
+        ignored << 'search' unless filter&.search.nil? || filter.search.to_s.empty?
+        return if ignored.empty?
+
+        ForestAdminDatasourcePylon.logger.warn(
+          "[forest_admin_datasource_pylon] PylonIssue ignored the #{ignored.join(" and ")} of this query; " \
+          'filtering is not implemented yet, so the returned records are unfiltered.'
+        )
+      end
+
+      def walker
+        @walker ||= Pagination::CursorWalker.new
+      end
+    end
+  end
+end

packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/schema_definition.rb

@@ -0,0 +1,76 @@
+module ForestAdminDatasourcePylon
+  module Collections
+    class Issue < BaseCollection
+      # Every column is read-only and non-sortable in this story: writes land in
+      # a later story, and `/issues/search` exposes no sort parameter at all —
+      # results always come back ordered by `created_at` descending, so
+      # advertising a sortable column would let the UI ask for an order the API
+      # cannot honour.
+      module SchemaDefinition
+        ColumnSchema = BaseCollection::ColumnSchema
+        Operators    = BaseCollection::Operators
+        STRING_OPS   = BaseCollection::STRING_OPS
+        NUMBER_OPS   = BaseCollection::NUMBER_OPS
+        DATE_OPS     = BaseCollection::DATE_OPS
+
+        private
+
+        def define_schema
+          define_identity_fields
+          define_content_fields
+          define_party_fields
+          define_time_fields
+        end
+
+        # Relations are declared in a later story, once the Account / Contact /
+        # User / Team collections exist to point at.
+        def define_relations; end
+
+        def define_identity_fields
+          # Only equal/in: these are the two the primary-key short-circuit can
+          # actually serve through GET /issues/{id}.
+          add_field('id', ColumnSchema.new(column_type: 'String',
+                                           filter_operators: [Operators::EQUAL, Operators::IN],
+                                           is_primary_key: true, is_read_only: true))
+          add_field('number', column('Number', NUMBER_OPS))
+          add_field('link', column('String', []))
+        end
+
+        def define_content_fields
+          add_field('title', column('String', STRING_OPS))
+          add_field('body_html', column('String', []))
+          # Left as String rather than Enum: Pylon ships five built-in states
+          # but organisations define their own on top of them.
+          add_field('state', column('String', STRING_OPS))
+          add_field('type', column('String', STRING_OPS))
+          add_field('source', column('String', STRING_OPS))
+          add_field('tags', column('Json', []))
+          add_field('customer_portal_visible', column('Boolean', []))
+          add_field('author_unverified', column('Boolean', []))
+          add_field('number_of_touches', column('Number', NUMBER_OPS))
+        end
+
+        # Flattened from the nested `{id: …}` objects Pylon returns. They stay
+        # plain columns until the story that adds the related collections.
+        def define_party_fields
+          %w[account_id requester_id assignee_id team_id].each do |field|
+            add_field(field, column('String', STRING_OPS))
+          end
+        end
+
+        def define_time_fields
+          %w[first_response_time resolution_time latest_message_time created_at updated_at].each do |field|
+            add_field(field, column('Date', DATE_OPS))
+          end
+          %w[time_in_status_seconds business_hours_time_in_status_seconds].each do |field|
+            add_field(field, column('Json', []))
+          end
+        end
+
+        def column(type, operators)
+          ColumnSchema.new(column_type: type, filter_operators: operators, is_read_only: true)
+        end
+      end
+    end
+  end
+end

packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/serializer.rb

@@ -0,0 +1,45 @@
+module ForestAdminDatasourcePylon
+  module Collections
+    class Issue < BaseCollection
+      module Serializer
+        PARTY_FIELDS = { 'account_id' => 'account', 'requester_id' => 'requester',
+                         'assignee_id' => 'assignee', 'team_id' => 'team' }.freeze
+
+        NATIVE_FIELDS = %w[id number link title body_html state type source tags
+                           customer_portal_visible author_unverified number_of_touches
+                           first_response_time resolution_time latest_message_time
+                           created_at updated_at time_in_status_seconds
+                           business_hours_time_in_status_seconds].freeze
+
+        private
+
+        def serialize(issue)
+          attrs = issue.is_a?(Hash) ? issue : {}
+          record = NATIVE_FIELDS.to_h { |field| [field, attrs[field]] }
+          PARTY_FIELDS.each { |column, source| record[column] = nested_id(attrs[source]) }
+          add_custom_field_values(record, attrs['custom_fields'])
+          record
+        end
+
+        def nested_id(value)
+          value['id'] if value.is_a?(Hash)
+        end
+
+        def add_custom_field_values(record, values)
+          custom_fields.each do |cf|
+            entry = values.is_a?(Hash) ? values[cf[:column_name]] : nil
+            record[cf[:column_name]] = custom_field_value(entry)
+          end
+        end
+
+        # Pylon spells a custom field as `slug => {"slug": ..., "value": ...}`,
+        # with `"values": [...]` instead of `"value"` for multi-value fields.
+        def custom_field_value(entry)
+          return entry unless entry.is_a?(Hash)
+
+          entry.key?('value') ? entry['value'] : entry['values']
+        end
+      end
+    end
+  end
+end

packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/datasource.rb

@@ -0,0 +1,19 @@
+module ForestAdminDatasourcePylon
+  class Datasource < ForestAdminDatasourceToolkit::Datasource
+    attr_reader :client, :configuration
+
+    def initialize(api_key:, **options)
+      super()
+      @configuration = Configuration.new(api_key: api_key, **options)
+      @client = Client.new(@configuration)
+
+      register_collections
+    end
+
+    private
+
+    def register_collections
+      add_collection(Collections::Issue.new(self))
+    end
+  end
+end