Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions lib/pgsync/task.rb
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,14 @@ def sync_data
where = "#{quote_ident(primary_key)} >= #{starting_id} AND #{quote_ident(primary_key)} < #{starting_id + batch_size}"
log " #{i}/#{batch_count}: #{where}"

# TODO be smarter for advance sql clauses
batch_sql_clause = " #{sql_clause.length > 0 ? "#{sql_clause} AND" : "WHERE"} #{where}"
# Apply the batch range to the complete filter, not just its last OR term.
batch_sql_clause =
if sql_clause.length > 0
where_clause = sql_clause.sub(/\A\s*where\b/i, "")
" WHERE (#{where_clause}) AND (#{where})"
else
" WHERE #{where}"
end

batch_copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quoted_table}#{batch_sql_clause}) TO STDOUT"
copy(batch_copy_to_command, dest_table: table, dest_fields: fields)
Expand Down
12 changes: 12 additions & 0 deletions test/in_batches_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,18 @@ def test_works
assert_result("--in-batches --batch-size 1", source, dest, expected)
end

def test_where_clause_with_or
source = 3.times.map { |i| {"id" => i + 1, "title" => "Post #{i + 1}"} }
expected = [source[0], source[2]]
assert_result("\"where id = 1 OR id = 3\" --in-batches --batch-size 1", source, [], expected)
end

def test_where_clause_without_space
source = 3.times.map { |i| {"id" => i + 1, "title" => "Post #{i + 1}"} }
expected = [source[0], source[2]]
assert_result("\"where(id = 1 OR id = 3)\" --in-batches --batch-size 1", source, [], expected)
end

def test_existing_data
source = 3.times.map { |i| {"id" => i + 1, "title" => "Post #{i + 1}"} }
dest = [{"id" => 1, "title" => "First Post"}, {"id" => 4, "title" => "Post 4"}]
Expand Down