diff --git a/.ameba.yml b/.ameba.yml index 16f60d9..9217d19 100644 --- a/.ameba.yml +++ b/.ameba.yml @@ -27,3 +27,7 @@ Style/ParenthesesAroundCondition: Description: Disallows redundant parentheses around control expressions Enabled: false Severity: Convention + +Lint/SpecFilename: + Excluded: + - spec/migration/**/*.cr diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf8ce06..db100be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,9 @@ name: CI on: push jobs: + test: + uses: PlaceOS/.github/.github/workflows/containerised-test.yml@main + crystal-style: uses: PlaceOS/.github/.github/workflows/crystal-style.yml@main diff --git a/.gitignore b/.gitignore index bd78e58..5505246 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ docs lib sam.cr *.swp +coverage diff --git a/README.md b/README.md index 90cb662..3373739 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,12 @@ docker-compose run --no-deps -it init task db:restore path=DUMP_FILE_LOCATION ho docker-compose run --no-deps -it init task migrate:rethink_dump path=DUMP_FILE_LOCATION host=$PG_HOST port=$PG_PORT db=$PG_DB user=$PG_USER password=$PG_PASSWORD clean_before=true ``` +```bash +# Migrate desks from level zone `desks` metadata to assets, remap desk bookings and enable `desk.use_assets` +# runs in a single transaction and is safe to re-run, `dry_run=true` prints the summary and rolls back +docker-compose run --no-deps -it init task migrate:desk_assets dry_run=true +``` + ```bash # Create a set of placeholder records docker-compose run --no-deps -it init task create:placeholder diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..05c0ae0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,53 @@ +x-postgresdb-client-env: &postgresdb-client-env + PG_HOST: ${PG_HOST:-postgres} + PG_PORT: ${PG_PORT:-5432} + PG_DB: ${PG_DB:-place_development} + PG_USER: ${PG_USER:-postgres} + PG_PASSWORD: ${PG_PASSWORD:-password} + PG_DATABASE: ${PG_DATABASE:-place_development} + PG_DATABASE_URL: ${PG_DATABASE_URL:-postgresql://postgres:password@postgres:5432/place_development} + +services: + test: + image: placeos/service-spec-runner:${CRYSTAL_VERSION:-latest} + volumes: + - ${PWD}/spec:/app/spec + - ${PWD}/src:/app/src + - ${PWD}/lib:/app/lib + - ${PWD}/shard.lock:/app/shard.lock + - ${PWD}/shard.override.yml:/app/shard.override.yml + - ${PWD}/shard.yml:/app/shard.yml.input + - ${PWD}/coverage:/app/coverage + depends_on: + postgres: + condition: service_healthy + migrator: + condition: service_completed_successfully + security_opt: + - seccomp:unconfined + environment: + GITHUB_ACTION: ${GITHUB_ACTION:-} + <<: *postgresdb-client-env + + postgres: + hostname: postgres + image: postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 30s + retries: 10 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: place_development + + migrator: + build: + context: ${PWD}/spec/migration + depends_on: + postgres: + condition: service_healthy + environment: + GITHUB_ACTION: ${GITHUB_ACTION:-} + PG_DATABASE_URL: ${PG_DATABASE_URL:-postgresql://postgres:password@postgres:5432/place_development} diff --git a/spec/desk_assets_spec.cr b/spec/desk_assets_spec.cr new file mode 100644 index 0000000..0878dcb --- /dev/null +++ b/spec/desk_assets_spec.cr @@ -0,0 +1,264 @@ +require "./spec_helper" + +module PlaceOS::Tasks + alias Generator = Model::Generator + + def self.zone(tag : String, parent : Model::Zone? = nil) : Model::Zone + zone = Model::Zone.new(name: "#{tag}-#{Random::Secure.hex(4)}", tags: Set{tag}) + zone.parent_id = parent.id if parent + zone.save! + end + + def self.domain(domain : String, org : Model::Zone?) : Model::Authority + authority = Generator.authority(domain) + authority.config["org_zone"] = JSON::Any.new(org.id.as(String)) if org + authority.save! + end + + def self.desks(level : Model::Zone, json : String) : Model::Metadata + Model::Metadata.new(name: "desks", parent_id: level.id.as(String), details: JSON.parse(json)).save! + end + + def self.desk_booking(tenant : Model::Tenant, desk_id : String, zones : Array(String)) : Model::Booking + booking = Generator.booking(tenant.id, [desk_id], 1.day.ago, 1.day.ago + 1.hour, booking_type: "desk") + booking.zones = zones + booking.save! + end + + def self.desk_assets : Array(Model::Asset) + Model::Asset.all.to_a.sort_by!(&.zone_id.to_s) + end + + def self.settings(org : Model::Zone) : JSON::Any? + Model::Metadata.for(org.id.as(String), "settings").first?.try(&.details) + end + + EXAMPLE_DESKS = <<-JSON + [ + { + "id": "desk-6.06.1", + "name": "6.06.01", + "notes": "", + "groups": ["staff"], + "images": ["https://example.com/desk.png"], + "map_id": "desk-6.06.1", + "bookable": true, + "features": ["Standard Office"], + "security": "DWN 6.06A", + "assigned_to": "Charles.Yeo@cdu.edu.au", + "assigned_name": "Charles Yeo" + }, + { + "id": "desk-6.06.2", + "name": "", + "notes": "by the window", + "groups": [], + "images": [], + "map_id": "", + "bookable": false, + "features": ["Standard Office", 3], + "security": "", + "assigned_to": "" + }, + {"id": "desk-6.06.3", "security": null}, + {"name": "no id"}, + {"id": "desk-6.06.4", "map_id": "desk-6.06.1"} + ] + JSON + + describe DeskAssets do + it "migrates desk metadata into assets for each domain" do + org = zone("org") + region = zone("region", org) + building = zone("building", region) + level = zone("level", building) + domain("https://one.example.com", org) + desks(level, EXAMPLE_DESKS) + + # domain without a region + org2 = zone("org") + building2 = zone("building", org2) + level2 = zone("level", building2) + domain("https://two.example.com", org2) + desks(level2, %([{"id": "desk-1"}])) + + # domain without an org_zone is ignored + ignored = zone("org") + domain("https://ignored.example.com", nil) + desks(zone("level", zone("building", ignored)), %([{"id": "desk-2"}])) + + summary = DeskAssets.migrate_desk_assets + summary.committed?.should be_true + summary.skipped_authorities.should eq ["ignored.example.com"] + + first = summary.domains.find!(&.org_zone_id.==(org.id)) + first.created.should eq 3 + first.skipped.size.should eq 2 + + assets = Model::Asset.where(zone_id: level.id.as(String)).to_a.to_h { |asset| {asset.map_id.as(String), asset} } + assets.keys.sort!.should eq ["desk-6.06.1", "desk-6.06.2", "desk-6.06.3"] + + desk = assets["desk-6.06.1"] + desk.identifier.should eq "6.06.01" + desk.zones.should eq [org.id, region.id, building.id, level.id] + desk.bookable.should be_true + desk.assigned_to.should eq "Charles.Yeo@cdu.edu.au" + desk.assigned_name.should eq "Charles Yeo" + desk.place_groups.should eq ["staff"] + desk.features.should eq ["Standard Office"] + desk.images.should eq ["https://example.com/desk.png"] + desk.security_system_groups.should eq ["DWN 6.06A"] + + desk = assets["desk-6.06.2"] + desk.identifier.should eq "desk-6.06.2" + desk.bookable.should be_false + desk.assigned_to.should be_nil + desk.notes.should eq "by the window" + desk.features.should eq ["Standard Office"] + desk.security_system_groups.should be_empty + + desk = assets["desk-6.06.3"] + desk.identifier.should eq "desk-6.06.3" + desk.bookable.should be_true + desk.security_system_groups.should be_empty + + desk = Model::Asset.find_by(zone_id: level2.id.as(String)) + desk.zones.should eq [org2.id, building2.id, level2.id] + desk.map_id.should eq "desk-1" + Model::Asset.count.should eq 4 + + # one category and type per migrated authority + Model::AssetCategory.where(name: DeskAssets::DESKS).count.should eq 2 + Model::AssetType.where(name: DeskAssets::DESKS).count.should eq 2 + end + + it "reuses an existing _DESKS_ category and type" do + org = zone("org") + authority = domain("https://one.example.com", org) + desks(zone("level", zone("building", org)), %([{"id": "desk-1"}])) + + category = Model::AssetCategory.new(name: DeskAssets::DESKS, authority_id: authority.id).save! + asset_type = Model::AssetType.new(name: DeskAssets::DESKS, brand: "existing", category_id: category.id).save! + + DeskAssets.migrate_desk_assets + Model::AssetCategory.count.should eq 1 + Model::AssetType.count.should eq 1 + Model::Asset.find_by(map_id: "desk-1").asset_type_id.should eq asset_type.id + end + + it "remaps desk bookings to the new asset ids" do + tenant = Generator.tenant + org = zone("org") + building = zone("building", org) + level1 = zone("level", building) + level2 = zone("level", building) + domain("https://one.example.com", org) + desks(level1, %([{"id": "desk-1"}, {"id": "shared"}])) + desks(level2, %([{"id": "shared"}])) + + booking = desk_booking(tenant, "desk-1", [org.id.as(String), building.id.as(String), level1.id.as(String)]) + shared = desk_booking(tenant, "shared", [org.id.as(String), building.id.as(String), level2.id.as(String)]) + stale = desk_booking(tenant, "desk-removed", [org.id.as(String), level1.id.as(String)]) + other = Generator.booking(tenant.id, "desk-1", 1.day.ago, 1.day.ago + 1.hour, booking_type: "parking").save! + + # an occurrence of a recurring booking moved to another desk + PgORM::Database.exec_sql(<<-SQL, booking.id, booking.booking_start + 1.week.total_seconds.to_i64, tenant.id, booking.booking_start, booking.booking_end) + INSERT INTO "booking_instances" (id, instance_start, tenant_id, booking_start, booking_end, asset_id, asset_ids, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, 'shared', '{shared}', NOW(), NOW()) + SQL + + summary = DeskAssets.migrate_desk_assets + summary.bookings.total.should eq 3 + summary.bookings.remapped.should eq 2 + summary.bookings.missed.should eq 1 + summary.bookings.missed_ids.should eq Set{"desk-removed"} + summary.instances.remapped.should eq 1 + + desk1 = Model::Asset.find_by(zone_id: level1.id.as(String), map_id: "desk-1") + shared_on_level2 = Model::Asset.find_by(zone_id: level2.id.as(String), map_id: "shared") + shared_on_level1 = Model::Asset.find_by(zone_id: level1.id.as(String), map_id: "shared") + + booking = Model::Booking.find(booking.id) + booking.asset_id.should eq desk1.id + booking.asset_ids.should eq [desk1.id] + + shared = Model::Booking.find(shared.id) + shared.asset_id.should eq shared_on_level2.id + shared.asset_ids.should eq [shared_on_level2.id] + + Model::Booking.find(stale.id).asset_id.should eq "desk-removed" + Model::Booking.find(other.id).asset_id.should eq "desk-1" + + # the instance is resolved using the parent booking zones (level1) + PgORM::Database.connection do |db| + db.query_one(%(SELECT asset_id FROM "booking_instances" WHERE id = $1), booking.id, as: String) + end.should eq shared_on_level1.id + end + + it "merges use_assets into the org zone settings" do + org = zone("org") + domain("https://one.example.com", org) + org2 = zone("org") + domain("https://two.example.com", org2) + Model::Metadata.new( + name: "settings", + parent_id: org2.id.as(String), + details: JSON.parse(%({"app": {"title": "Place"}, "desk": {"auto_release": true}})) + ).save! + + DeskAssets.migrate_desk_assets + settings(org).should eq JSON.parse(%({"desk": {"use_assets": true}})) + settings(org2).should eq JSON.parse(%({"app": {"title": "Place"}, "desk": {"auto_release": true, "use_assets": true}})) + end + + it "can be run multiple times" do + tenant = Generator.tenant + org = zone("org") + level = zone("level", zone("building", org)) + domain("https://one.example.com", org) + metadata = desks(level, %([{"id": "desk-1", "name": "One"}, {"id": "desk-2"}])) + desk_booking(tenant, "desk-1", [org.id.as(String), level.id.as(String)]) + desk_booking(tenant, "desk-2", [org.id.as(String), level.id.as(String)]) + + first = DeskAssets.migrate_desk_assets + first.bookings.remapped.should eq 2 + ids = desk_assets.map(&.id) + + # fix some data and run again + metadata.details = JSON.parse(%([{"id": "desk-1", "name": "Desk One"}, {"id": "desk-2"}])) + metadata.save! + + second = DeskAssets.migrate_desk_assets + second.domains.first.created.should eq 0 + second.domains.first.updated.should eq 2 + second.domains.first.settings.should eq "unchanged" + second.bookings.remapped.should eq 0 + second.bookings.missed.should eq 0 + second.bookings.already_migrated.should eq 2 + + desk_assets.map(&.id).should eq ids + Model::Asset.find_by(map_id: "desk-1").identifier.should eq "Desk One" + Model::AssetCategory.count.should eq 1 + Model::AssetType.count.should eq 1 + end + + it "rolls back a dry run" do + tenant = Generator.tenant + org = zone("org") + level = zone("level", zone("building", org)) + domain("https://one.example.com", org) + desks(level, %([{"id": "desk-1"}])) + booking = desk_booking(tenant, "desk-1", [org.id.as(String), level.id.as(String)]) + + summary = DeskAssets.migrate_desk_assets(dry_run: true) + summary.committed?.should be_false + summary.domains.first.created.should eq 1 + summary.bookings.remapped.should eq 1 + + Model::Asset.count.should eq 0 + Model::AssetCategory.count.should eq 0 + Model::Booking.find(booking.id).asset_id.should eq "desk-1" + settings(org).should be_nil + end + end +end diff --git a/spec/migration/Dockerfile b/spec/migration/Dockerfile new file mode 100644 index 0000000..11531ab --- /dev/null +++ b/spec/migration/Dockerfile @@ -0,0 +1,15 @@ +FROM placeos/crystal:latest + +WORKDIR /app + +COPY . /app + +# NOTE:: add `--branch my-new-model-branch` if you want to test a particular branch +RUN git clone https://github.com/PlaceOS/models +RUN mv ./models/migration/db ./db + +ENV PATH /app/bin:$PATH + +RUN shards build + +ENTRYPOINT [ "/app/run.sh" ] diff --git a/spec/migration/LICENSE b/spec/migration/LICENSE new file mode 100644 index 0000000..d57fad6 --- /dev/null +++ b/spec/migration/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Ali Naqvi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/spec/migration/README.md b/spec/migration/README.md new file mode 100644 index 0000000..c43d139 --- /dev/null +++ b/spec/migration/README.md @@ -0,0 +1,3 @@ +#PlaceOS Models Migration Script + +This folder contains source code for scaffolding PlaceOS models schema and running migration. diff --git a/spec/migration/run.sh b/spec/migration/run.sh new file mode 100755 index 0000000..5b76a70 --- /dev/null +++ b/spec/migration/run.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -eu + +bin/migration up \ No newline at end of file diff --git a/spec/migration/shard.lock b/spec/migration/shard.lock new file mode 100644 index 0000000..33adf43 --- /dev/null +++ b/spec/migration/shard.lock @@ -0,0 +1,14 @@ +version: 2.0 +shards: + db: + git: https://github.com/crystal-lang/crystal-db.git + version: 0.10.1 + + micrate: + git: https://github.com/amberframework/micrate.git + version: 0.12.0 + + pg: + git: https://github.com/will/crystal-pg.git + version: 0.25.0 + diff --git a/spec/migration/shard.yml b/spec/migration/shard.yml new file mode 100644 index 0000000..ae3873b --- /dev/null +++ b/spec/migration/shard.yml @@ -0,0 +1,16 @@ +name: placeos-models-migration +version: 0.9.9 +crystal: ~> 1.4 +license: MIT + + +targets: + migration: + main: src/migration.cr + +dependencies: + micrate: + github: amberframework/micrate + version: "0.12.0" + pg: + github: will/crystal-pg diff --git a/spec/migration/src/migration.cr b/spec/migration/src/migration.cr new file mode 100644 index 0000000..d59fe70 --- /dev/null +++ b/spec/migration/src/migration.cr @@ -0,0 +1,5 @@ +require "micrate" +require "pg" + +Micrate::DB.connection_url = ENV["PG_DATABASE_URL"] +Micrate::Cli.run diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr new file mode 100644 index 0000000..200f739 --- /dev/null +++ b/spec/spec_helper.cr @@ -0,0 +1,20 @@ +require "spec" +require "placeos-models" +require "placeos-models/spec/generator" +require "../src/tasks/desk_assets" + +PgORM::Database.configure { |_| } + +def clear_tables + PgORM::Database.exec_sql <<-SQL + TRUNCATE "asset", "asset_type", "asset_category", "metadata", "zone", + "booking_instances", "bookings", "tenants", "authority", "history" CASCADE + SQL +end + +Spec.before_suite do + Log.setup(:warn) + clear_tables +end + +Spec.before_each { clear_tables } diff --git a/src/sam.cr b/src/sam.cr index 250055e..06aed5e 100644 --- a/src/sam.cr +++ b/src/sam.cr @@ -213,6 +213,20 @@ namespace "migrate" do PlaceOS::Tasks.migrate_rethink_to_pg(**arguments) end + + desc "Migrate desks from level zone metadata to assets, remapping desk bookings. `dry_run=true` rolls back" + task "desk_assets" do |_, args| + arguments = { + host: (args["host"]? || PlaceOS::PG_HOST).to_s, + port: (args["port"]? || PlaceOS::PG_PORT).to_i, + db: (args["db"]? || PlaceOS::PG_DB).to_s, + user: (args["user"]? || PlaceOS::PG_USER).try &.to_s, + password: (args["password"]? || PlaceOS::PG_PASS).try &.to_s, + dry_run: args["dry_run"]?.try(&.to_s.downcase) == "true", + } + + PlaceOS::Tasks.migrate_desk_assets(**arguments) + end end desc "Drops the PostgreSQL DB" diff --git a/src/tasks.cr b/src/tasks.cr index b136aa7..212fe0b 100644 --- a/src/tasks.cr +++ b/src/tasks.cr @@ -6,6 +6,7 @@ module PlaceOS::Tasks include Backup include Database + include DeskAssets include Entities include Initialization include Restore diff --git a/src/tasks/desk_assets.cr b/src/tasks/desk_assets.cr new file mode 100644 index 0000000..506e447 --- /dev/null +++ b/src/tasks/desk_assets.cr @@ -0,0 +1,408 @@ +require "json" +require "pg-orm" +require "placeos-models" + +# Migrates desks from the `desks` metadata on level zones into the asset table, +# remaps desk bookings to the new asset ids and enables `desk.use_assets` in the +# org zone settings. Everything is written in a single transaction and the task +# can be re-run: existing assets are matched on (asset type, zone_id, map_id). +module PlaceOS::Tasks::DeskAssets + extend self + Log = ::Log.for(self) + + DESKS = "_DESKS_" + BATCH_SIZE = 1000 + + record Desk, + legacy_id : String, + map_id : String, + identifier : String, + zone_id : String, + zones : Array(String), + bookable : Bool, + assigned_to : String?, + assigned_name : String?, + place_groups : Array(String), + features : Array(String), + images : Array(String), + notes : String?, + security_system_groups : Array(String) + + # The desks collected for a single authority + class Domain + getter authority_id : String + getter domain : String + getter org_zone_id : String + property levels : Int32 = 0 + property levels_with_desks : Int32 = 0 + getter desks = [] of Desk + getter skipped = [] of String + property created : Int32 = 0 + property updated : Int32 = 0 + property settings : String = "unchanged" + + def initialize(@authority_id, @domain, @org_zone_id) + end + end + + class RemapStats + property total : Int32 = 0 + property remapped : Int32 = 0 + property already_migrated : Int32 = 0 + property missed : Int32 = 0 + getter missed_ids = Set(String).new + end + + class Summary + getter domains = [] of Domain + getter skipped_authorities = [] of String + getter bookings = RemapStats.new + getter instances = RemapStats.new + property? dry_run : Bool = false + property? committed : Bool = false + + def to_s(io : IO) : Nil + io << "Desk asset migration " << (dry_run? ? "(dry run, rolled back)" : (committed? ? "(committed)" : "(not committed)")) << '\n' + io << " authorities skipped (no org_zone): " << skipped_authorities.size << '\n' + domains.each do |domain| + io << " " << domain.domain << " (" << domain.authority_id << ", org " << domain.org_zone_id << ")\n" + io << " levels: " << domain.levels << ", with desks: " << domain.levels_with_desks << '\n' + io << " desks: " << domain.desks.size << ", created: " << domain.created << ", updated: " << domain.updated << ", skipped: " << domain.skipped.size << '\n' + domain.skipped.each { |reason| io << " skipped " << reason << '\n' } + io << " settings: " << domain.settings << '\n' + end + write_stats(io, "desk bookings", bookings) + write_stats(io, "booking instances", instances) + end + + private def write_stats(io, label, stats) + io << " " << label << ": " << stats.total << ", remapped: " << stats.remapped + io << ", already migrated: " << stats.already_migrated << ", missed: " << stats.missed << '\n' + unless stats.missed_ids.empty? + io << " missed desk ids (" << stats.missed_ids.size << "): " << stats.missed_ids.first(20).join(", ") + io << ", ..." if stats.missed_ids.size > 20 + io << '\n' + end + end + end + + def migrate_desk_assets(host : String, port : Int32, db : String, user : String?, password : String?, dry_run : Bool = false) : Summary + PgORM::Database.configure do |settings| + settings.host = host + settings.port = port + settings.db = db + settings.user = user || "postgres" + settings.password = password || "" + end + + migrate_desk_assets(dry_run).tap { |summary| puts summary } + end + + def migrate_desk_assets(dry_run : Bool = false) : Summary + summary = Summary.new + summary.dry_run = dry_run + + # 1 & 2. collect and prepare + collect(summary) + + # 3. migrate + PgORM::Database.transaction do + lookup = {} of String => Array(Candidate) + desk_asset_ids = Set(String).new + + summary.domains.each do |domain| + migrate_domain(domain, lookup, desk_asset_ids) + enable_assets_setting(domain) + end + + remap_bookings(lookup, desk_asset_ids, summary.bookings) + remap_instances(lookup, desk_asset_ids, summary.instances) + + raise DB::Rollback.new if dry_run + summary.committed = true + end + + summary + rescue error + Log.error(exception: error) { "desk asset migration failed, rolled back" } + raise error + end + + # Collect + ############################################################################################### + + private record ZoneNode, id : String, parent_id : String?, tags : Array(String) + + protected def collect(summary : Summary) : Nil + zones = load_zones + children = Hash(String, Array(ZoneNode)).new { |hash, key| hash[key] = [] of ZoneNode } + zones.each_value { |zone| zone.parent_id.try { |parent_id| children[parent_id] << zone } } + + Model::Authority.all.each do |authority| + org_zone_id = authority.config["org_zone"]?.try(&.as_s?).presence + unless org_zone_id + summary.skipped_authorities << authority.domain + next + end + + domain = Domain.new(authority.id.as(String), authority.domain, org_zone_id) + summary.domains << domain + + unless zones.has_key?(org_zone_id) + domain.skipped << "org zone #{org_zone_id}: not found" + next + end + + levels = {} of String => Array(String) + find_levels(zones[org_zone_id], [] of String, children, levels, Set(String).new) + domain.levels = levels.size + + levels.each do |level_id, branch| + next unless metadata = Model::Metadata.for(level_id, "desks").first? + desks = metadata.details.as_a? + unless desks + domain.skipped << "level #{level_id}: desks metadata is not an array" + next + end + + domain.levels_with_desks += 1 + map_ids = Set(String).new + desks.each_with_index do |json, index| + desk = parse_desk(json, level_id, branch) + if desk.is_a?(String) + domain.skipped << "level #{level_id} desk #{index}: #{desk}" + elsif !map_ids.add?(desk.map_id) + domain.skipped << "level #{level_id} desk #{desk.legacy_id}: duplicate map_id #{desk.map_id}" + else + domain.desks << desk + end + end + end + end + end + + private def load_zones : Hash(String, ZoneNode) + zones = {} of String => ZoneNode + PgORM::Database.connection do |db| + db.query_each(%(SELECT id, parent_id, tags FROM "zone")) do |rs| + zone = ZoneNode.new(rs.read(String), rs.read(String?), rs.read(Array(String)?) || [] of String) + zones[zone.id] = zone + end + end + zones + end + + # depth first walk of the zone tree, levels are the leaves we are interested in + private def find_levels(zone : ZoneNode, path : Array(String), children, levels, visited) : Nil + return unless visited.add?(zone.id) + branch = path + [zone.id] + + if zone.tags.includes?("level") + levels[zone.id] = branch + return + end + + children[zone.id]?.try &.each { |child| find_levels(child, branch, children, levels, visited) } + end + + # Prepare + ############################################################################################### + + # returns the reason for skipping when the desk can't be migrated + protected def parse_desk(json : JSON::Any, level_id : String, branch : Array(String)) : Desk | String + desk = json.as_h? + return "not an object" unless desk + + id = string(desk["id"]?) + map_id = string(desk["map_id"]?) || id + return "missing id and map_id" unless map_id + legacy_id = id || map_id + + Desk.new( + legacy_id: legacy_id, + map_id: map_id, + identifier: string(desk["name"]?) || legacy_id, + zone_id: level_id, + zones: branch, + bookable: desk["bookable"]?.try(&.as_bool?) != false, + assigned_to: string(desk["assigned_to"]?), + assigned_name: string(desk["assigned_name"]?), + place_groups: strings(desk["groups"]?), + features: strings(desk["features"]?), + images: strings(desk["images"]?), + notes: string(desk["notes"]?), + security_system_groups: string(desk["security"]?).try { |group| [group] } || [] of String, + ) + end + + private def string(value : JSON::Any?) : String? + value.try(&.as_s?).presence + end + + private def strings(value : JSON::Any?) : Array(String) + value.try(&.as_a?).try(&.compact_map(&.as_s?)) || [] of String + end + + # Migrate + ############################################################################################### + + private def migrate_domain(domain : Domain, lookup, desk_asset_ids) : Nil + asset_type = desks_asset_type(domain.authority_id) + type_id = asset_type.id.as(String) + + existing = Model::Asset.where(asset_type_id: type_id).to_a + existing.each { |asset| desk_asset_ids << asset.id.as(String) } + index = existing.to_h { |asset| { {asset.zone_id, asset.map_id}, asset } } + + domain.desks.each do |desk| + asset = index[{desk.zone_id, desk.map_id}]? + if asset + domain.updated += 1 + else + asset = Model::Asset.new + asset.asset_type_id = type_id + asset.zone_id = desk.zone_id + asset.map_id = desk.map_id + domain.created += 1 + end + + asset.identifier = desk.identifier + asset.zones = desk.zones + asset.bookable = desk.bookable + asset.assigned_to = desk.assigned_to + asset.assigned_name = desk.assigned_name + asset.place_groups = desk.place_groups + asset.features = desk.features + asset.images = desk.images + asset.notes = desk.notes + asset.security_system_groups = desk.security_system_groups + asset.save! + + asset_id = asset.id.as(String) + desk_asset_ids << asset_id + (lookup[desk.legacy_id] ||= [] of Candidate) << Candidate.new(asset_id, desk.zone_id) + end + end + + private def desks_asset_type(authority_id : String) : Model::AssetType + category = Model::AssetCategory.find_by?(authority_id: authority_id, name: DESKS) || + Model::AssetCategory.new(name: DESKS, authority_id: authority_id).save! + + category_id = category.id.as(String) + Model::AssetType.find_by?(category_id: category_id, name: DESKS) || + Model::AssetType.new(name: DESKS, brand: DESKS, category_id: category_id).save! + end + + private def enable_assets_setting(domain : Domain) : Nil + metadata = Model::Metadata.for(domain.org_zone_id, "settings").first? + domain.settings = metadata ? "updated" : "created" + metadata ||= Model::Metadata.new(name: "settings", parent_id: domain.org_zone_id, details: JSON::Any.new({} of String => JSON::Any)) + + # copies, mutating the existing details in place hides the change from dirty tracking + details = metadata.details.as_h?.try(&.dup) || {} of String => JSON::Any + desk = details["desk"]?.try(&.as_h?).try(&.dup) + unless desk + Log.warn { "#{domain.org_zone_id} settings.desk is not an object, replacing it" } if details.has_key?("desk") + desk = {} of String => JSON::Any + end + + if desk["use_assets"]?.try(&.as_bool?) == true + domain.settings = "unchanged" + return + end + + desk["use_assets"] = JSON::Any.new(true) + details["desk"] = JSON::Any.new(desk) + metadata.details = JSON::Any.new(details) + metadata.save! + end + + # Remap bookings + ############################################################################################### + + # an asset created for a legacy desk id, desk ids are only unique within a level + private record Candidate, asset_id : String, level_id : String + + private record Row, key : JSON::Any, asset_id : String?, asset_ids : Array(String), zones : Array(String) + + private def remap_bookings(lookup, desk_asset_ids, stats : RemapStats) : Nil + rows = [] of Row + PgORM::Database.connection do |db| + db.query_each(%(SELECT id, asset_id, asset_ids, zones FROM "bookings" WHERE booking_type = 'desk')) do |rs| + key = JSON::Any.new({"id" => JSON::Any.new(rs.read(Int64))}) + rows << Row.new(key, rs.read(String?), rs.read(Array(String)?) || [] of String, rs.read(Array(String)?) || [] of String) + end + end + + remap(rows, lookup, desk_asset_ids, stats, <<-SQL) + UPDATE "bookings" AS b SET asset_id = u.asset_id, asset_ids = u.asset_ids + FROM jsonb_to_recordset($1::jsonb) AS u(id bigint, asset_id text, asset_ids text[]) + WHERE b.id = u.id + SQL + end + + # occurrences of recurring desk bookings can override the parent booking assets + private def remap_instances(lookup, desk_asset_ids, stats : RemapStats) : Nil + rows = [] of Row + PgORM::Database.connection do |db| + db.query_each(<<-SQL) do |rs| + SELECT bi.id, bi.instance_start, bi.asset_id, bi.asset_ids, b.zones + FROM "booking_instances" AS bi INNER JOIN "bookings" AS b ON b.id = bi.id + WHERE b.booking_type = 'desk' AND (bi.asset_id IS NOT NULL OR bi.asset_ids IS NOT NULL) + SQL + key = JSON::Any.new({"id" => JSON::Any.new(rs.read(Int64)), "instance_start" => JSON::Any.new(rs.read(Int64))}) + rows << Row.new(key, rs.read(String?), rs.read(Array(String)?) || [] of String, rs.read(Array(String)?) || [] of String) + end + end + + remap(rows, lookup, desk_asset_ids, stats, <<-SQL) + UPDATE "booking_instances" AS bi SET asset_id = u.asset_id, asset_ids = u.asset_ids + FROM jsonb_to_recordset($1::jsonb) AS u(id bigint, instance_start bigint, asset_id text, asset_ids text[]) + WHERE bi.id = u.id AND bi.instance_start = u.instance_start + SQL + end + + private def remap(rows : Array(Row), lookup, desk_asset_ids, stats : RemapStats, update_sql : String) : Nil + updates = [] of JSON::Any + + rows.each do |row| + stats.total += 1 + legacy_ids = row.asset_ids.empty? ? [row.asset_id].compact : row.asset_ids + if legacy_ids.empty? + stats.missed += 1 + next + end + + if legacy_ids.all? { |id| desk_asset_ids.includes?(id) } + stats.already_migrated += 1 + next + end + + asset_ids = legacy_ids.map { |id| resolve(id, row.zones, lookup, desk_asset_ids) } + if missing = asset_ids.index(nil) + stats.missed += 1 + stats.missed_ids << legacy_ids[missing] + next + end + + resolved = asset_ids.compact + update = row.key.as_h.dup + update["asset_id"] = JSON::Any.new(resolved.first) + update["asset_ids"] = JSON::Any.new(resolved.map { |id| JSON::Any.new(id) }) + updates << JSON::Any.new(update) + stats.remapped += 1 + end + + updates.each_slice(BATCH_SIZE) do |batch| + PgORM::Database.connection { |db| db.exec(update_sql, args: [batch.to_json]) } + end + end + + # prefer the candidate on one of the booking zones when a desk id exists on multiple levels + private def resolve(id : String, zones : Array(String), lookup, desk_asset_ids) : String? + return id if desk_asset_ids.includes?(id) + return unless candidates = lookup[id]? + return candidates.first.asset_id if candidates.size == 1 + candidates.find { |candidate| zones.includes?(candidate.level_id) }.try &.asset_id + end +end diff --git a/test b/test new file mode 100755 index 0000000..815052c --- /dev/null +++ b/test @@ -0,0 +1,29 @@ +#! /usr/bin/env bash + +set -eu + +# this function is called when Ctrl-C is sent +function trap_ctrlc () +{ + docker compose down &> /dev/null + exit 2 +} + +# initialise trap to call trap_ctrlc function +# when signal 2 (SIGINT) is received +trap "trap_ctrlc" 2 + +docker compose pull + +docker compose build + +exit_code="0" + +docker compose run \ + --rm \ + test "$@" \ + || exit_code="$?" + +docker compose down &> /dev/null + +exit ${exit_code}