feat: PPT-526 seed the management partner and own the placeholder estate

Dockerfile

@@ -81,7 +81,8 @@ RUN for binary in /app/bin/* /usr/bin/pg_dump /usr/bin/pg_restore /usr/bin/psql;
         xargs -I % sh -c 'mkdir -p $(dirname deps%); cp % deps%;'; \
     done
 
-RUN git clone https://github.com/PlaceOS/models
+# PPT-526: the tenancy migrations live on this branch until models #332 merges
+RUN git clone --branch PPT-526-partner-client https://github.com/PlaceOS/models
 
 # Create tmp directory with proper permissions
 RUN rm -rf /tmp && mkdir -p /tmp && chmod 1777 /tmp

shard.lock

@@ -143,7 +143,7 @@ shards:
 
   pg-orm:
     git: https://github.com/spider-gazelle/pg-orm.git
-    version: 2.5.2
+    version: 2.5.3
 
   place_calendar:
     git: https://github.com/placeos/calendar.git
@@ -153,9 +153,9 @@ shards:
     git: https://github.com/place-labs/log-backend.git
     version: 0.13.0
 
-  placeos-models:
+  placeos-models: # Overridden
     git: https://github.com/placeos/models.git
-    version: 9.123.3
+    version: 9.123.4+git.commit.b50200942c7e6bf30d999c1f500ce1375408edc0
 
   sam:
     git: https://github.com/imdrasil/sam.cr.git

shard.override.yml

@@ -1,4 +1,9 @@
 dependencies:
+  # PPT-526: build against the multi-tenancy data layer until models #332 merges
+  placeos-models:
+    github: placeos/models
+    branch: PPT-526-partner-client
+
   retriable:
     github: Sija/retriable.cr
   db:

spec/entities_spec.cr

@@ -0,0 +1,92 @@
+require "./spec_helper"
+require "../src/tasks/entities"
+
+module PlaceOS::Tasks
+  # The organisation the backfill migration creates for a domain that has none
+  def self.backfilled_organisation(authority : Model::Authority) : Model::Organisation
+    organisation = Model::Organisation.new(
+      name: "#{authority.name} (#{authority.domain})",
+      description: "Auto-created from authority #{authority.id} during PPT-526 organisation backfill",
+    ).save!
+    authority.organisation_id = organisation.id
+    authority.save!
+    organisation
+  end
+
+  def self.owned_zone(organisation : Model::Organisation) : Model::Zone
+    zone = Model::Zone.new(name: "zone-#{Random::Secure.hex(4)}", tags: Set{"org"})
+    zone.organisation_id = organisation.id
+    zone.save!
+  end
+
+  def self.organisation_grant(organisation : Model::Organisation) : Model::Grant
+    user = Model::Generator.user.save!
+    Model::Generator.grant(user, Model::Grant::SCOPE_ORGANISATION, organisation.id.to_s)
+  end
+
+  describe Entities do
+    describe "create_management_organisation" do
+      it "seeds the management partner and attaches an unowned domain to its staff organisation" do
+        authority = Model::Generator.authority("https://first.example.com").save!
+
+        organisation = Entities.create_management_organisation(authority)
+
+        organisation.partner_staff.should be_true
+        organisation.partner.not_nil!.management.should be_true
+        Model::Authority.find!(authority.id.as(String)).organisation_id.should eq organisation.id
+      end
+
+      it "is idempotent" do
+        authority = Model::Generator.authority("https://first.example.com").save!
+
+        first = Entities.create_management_organisation(authority)
+        second = Entities.create_management_organisation(Model::Authority.find!(authority.id.as(String)))
+
+        second.id.should eq first.id
+        Model::Partner.where(management: true).count.should eq 1
+        Model::Organisation.where(partner_staff: true).count.should eq 1
+      end
+
+      it "moves a backfilled estate into the staff organisation" do
+        authority = Model::Generator.authority("https://first.example.com").save!
+        backfilled = backfilled_organisation(authority)
+        zone = owned_zone(backfilled)
+        grant = organisation_grant(backfilled).save!
+
+        organisation = Entities.create_management_organisation(authority)
+
+        authority.organisation_id.should eq organisation.id
+        Model::Authority.find!(authority.id.as(String)).organisation_id.should eq organisation.id
+        Model::Zone.find!(zone.id.as(String)).organisation_id.should eq organisation.id
+        Model::Grant.find!(grant.id.not_nil!).scope_id.should eq organisation.id.to_s
+        Model::Organisation.find?(backfilled.id.not_nil!).should be_nil
+      end
+
+      it "leaves an organisation an operator has changed" do
+        authority = Model::Generator.authority("https://first.example.com").save!
+        backfilled = backfilled_organisation(authority)
+        backfilled.description = "Acme, merged by hand"
+        backfilled.save!
+        zone = owned_zone(backfilled)
+
+        Entities.create_management_organisation(authority)
+
+        Model::Authority.find!(authority.id.as(String)).organisation_id.should eq backfilled.id
+        Model::Zone.find!(zone.id.as(String)).organisation_id.should eq backfilled.id
+      end
+
+      it "leaves an organisation that owns other domains" do
+        authority = Model::Generator.authority("https://first.example.com").save!
+        backfilled = backfilled_organisation(authority)
+        other = Model::Generator.authority("https://second.example.com")
+        other.organisation_id = backfilled.id
+        other.save!
+
+        Entities.create_management_organisation(authority)
+
+        Model::Authority.find!(authority.id.as(String)).organisation_id.should eq backfilled.id
+        Model::Organisation.find?(backfilled.id.not_nil!).should_not be_nil
+      end
+    end
+  end
+end

spec/migration/Dockerfile

@@ -5,7 +5,8 @@ WORKDIR /app
 COPY . /app
 
 # NOTE:: add `--branch my-new-model-branch` if you want to test a particular branch
-RUN git clone https://github.com/PlaceOS/models
+# PPT-526: the tenancy migrations live on this branch until models #332 merges
+RUN git clone --branch PPT-526-partner-client https://github.com/PlaceOS/models
 RUN mv ./models/migration/db ./db
 
 ENV PATH /app/bin:$PATH

spec/spec_helper.cr

@@ -8,7 +8,8 @@ PgORM::Database.configure { |_| }
 def clear_tables
   PgORM::Database.exec_sql <<-SQL
     TRUNCATE "asset", "asset_type", "asset_category", "metadata", "zone",
-      "booking_instances", "bookings", "tenants", "shortener", "user", "authority", "history" CASCADE
+      "booking_instances", "bookings", "tenants", "shortener", "user", "authority", "history",
+      "grants", "partners" CASCADE
     SQL
 end
 

src/sam.cr

@@ -275,7 +275,9 @@ namespace "create" do
 
   desc "Creates a representative set of documents in PostgreSQL DB"
   task "placeholders" do
-    PlaceOS::Tasks.create_placeholders
+    # placeholders belong to the staff organisation when one exists
+    staff = PlaceOS::Model::Organisation.where(partner_staff: true).first?
+    PlaceOS::Tasks.create_placeholders(staff.try(&.id))
   end
 
   desc "Creates an authority"

src/tasks/entities.cr

@@ -35,6 +35,85 @@ module PlaceOS::Tasks::Entities
     raise e
   end
 
+  # Tables whose rows an organisation owns through `organisation_id`
+  OWNED_TABLES = {"authority", "zone", "sys", "mod", "trigger", "edge", "broker"}
+
+  # The platform operator's partner and staff organisation. The first domain
+  # belongs to that organisation, so its admins have cluster reach once
+  # tenancy enforcement is on.
+  #
+  # On a cluster that predates tenancy the backfill migration has already
+  # given the first domain an organisation of its own. While that organisation
+  # is untouched, its estate moves into the staff organisation.
+  def create_management_organisation(
+    authority : Model::Authority,
+    partner_name : String = "PlaceOS",
+  ) : Model::Organisation
+    partner = upsert_document(Model::Partner.where(management: true)) do
+      Log.info { {message: "creating management Partner", name: partner_name} }
+      Model::Partner.new(name: partner_name, description: "Platform operator (management partner)").tap do |new_partner|
+        new_partner.management = true
+      end
+    end
+
+    organisation = upsert_document(Model::Organisation.where(partner_id: partner.id, partner_staff: true)) do
+      Log.info { {message: "creating staff Organisation", name: authority.name} }
+      Model::Organisation.new(name: "#{partner_name} staff", description: "Staff of the management partner").tap do |new_org|
+        new_org.partner_id = partner.id
+        new_org.partner_staff = true
+        new_org.payer = Model::Organisation::PAYER_ORGANISATION
+      end
+    end
+
+    if authority.organisation_id.nil?
+      authority.organisation_id = organisation.id
+      authority.save!
+      Log.info { {message: "attached Authority to staff Organisation", authority: authority.id, organisation: organisation.id.to_s} }
+    elsif backfilled = backfilled_organisation(authority)
+      adopt_estate(from: backfilled, into: organisation)
+      authority.organisation_id = organisation.id
+      Log.info { {message: "moved the backfilled estate into the staff Organisation", authority: authority.id, from: backfilled.id.to_s, organisation: organisation.id.to_s} }
+    end
+
+    organisation
+  rescue e
+    log_fail("Organisation", e)
+    raise e
+  end
+
+  # The organisation the backfill migration created for this authority, while
+  # it is still as created: no partner, no other domain and the backfill's own
+  # description. Anything an operator has since changed is left alone.
+  protected def backfilled_organisation(authority : Model::Authority) : Model::Organisation?
+    organisation_id = authority.organisation_id
+    return unless organisation_id
+
+    organisation = Model::Organisation.find?(organisation_id)
+    return unless organisation
+    return unless organisation.partner_id.nil?
+    return unless organisation.description == "Auto-created from authority #{authority.id} during PPT-526 organisation backfill"
+    return unless organisation.authorities.count == 1
+
+    organisation
+  end
+
+  # Moves every row and grant owned by `from` to `into`, then removes `from`
+  protected def adopt_estate(from : Model::Organisation, into : Model::Organisation) : Nil
+    from_id = from.id.to_s
+    into_id = into.id.to_s
+
+    PgORM::Database.transaction do |tx|
+      OWNED_TABLES.each do |table|
+        tx.connection.exec(%(UPDATE "#{table}" SET organisation_id = $1::uuid WHERE organisation_id = $2::uuid), args: [into_id, from_id])
+      end
+      tx.connection.exec(
+        %(UPDATE "grants" SET scope_id = $1 WHERE scope_type = $2 AND scope_id = $3),
+        args: [into_id, Model::Grant::SCOPE_ORGANISATION, from_id],
+      )
+      from.destroy
+    end
+  end
+
   def create_interface(
     name : String,
     folder_name : String,
@@ -141,7 +220,7 @@ module PlaceOS::Tasks::Entities
     raise e
   end
 
-  def create_placeholders
+  def create_placeholders(organisation_id : UUID? = nil)
     version = UUID.random.to_s.split('-').first
 
     private_repository_uri = "https://github.com/placeos/private-drivers"
@@ -201,6 +280,7 @@ module PlaceOS::Tasks::Entities
         Model::Zone.new.tap do |zone|
           zone.name = "Zone-#{tag}-#{version}"
           zone.tags = Set{tag}
+          zone.organisation_id = organisation_id
         end
       end
     end
@@ -213,7 +293,9 @@ module PlaceOS::Tasks::Entities
     zones[2].save!
 
     control_system = upsert_document(Model::ControlSystem.all) do
-      Model::ControlSystem.new(name: "System-#{version}")
+      Model::ControlSystem.new(name: "System-#{version}").tap do |sys|
+        sys.organisation_id = organisation_id
+      end
     end
 
     upsert_document(Model::Settings.for_parent(control_system.id.as(String))) do
@@ -227,6 +309,7 @@ module PlaceOS::Tasks::Entities
     mod = upsert_document(Model::Module.where(driver_id: driver.id.as(String), control_system_id: control_system.id.as(String))) do
       Model::Generator.module(driver: driver, control_system: control_system).tap do |new_module|
         new_module.custom_name = "Module-#{version}"
+        new_module.organisation_id = organisation_id
       end
     end
 
@@ -241,6 +324,7 @@ module PlaceOS::Tasks::Entities
       trigger_description = "An automatically generated Trigger."
       new_trigger = Model::Trigger.new(name: trigger_name, description: trigger_description)
       new_trigger.control_system = control_system
+      new_trigger.organisation_id = organisation_id
       new_trigger
     end
 

src/tasks/initialization.cr

@@ -33,6 +33,7 @@ module PlaceOS::Tasks::Initialization
 
     start_lock.synchronize do
       authority = Entities.create_authority(name: application_base, domain: application_base, config: metrics_config)
+      organisation = Entities.create_management_organisation(authority)
 
       Entities.create_user(authority: authority, name: username, email: email, password: password, sys_admin: true)
       Entities.create_application(authority: authority, name: application_name, base: application_base)
@@ -53,7 +54,7 @@ module PlaceOS::Tasks::Initialization
 
       unless PlaceOS::Tasks.production? || PlaceOS::SKIP_PLACEHOLDERS
         Log.info { "creating placeholder documents" }
-        Entities.create_placeholders
+        Entities.create_placeholders(organisation.id)
       end
     end
   end