Browse Source

Merge branch 'main' into full-import-optimisation

full-import-optimisation
natdeanlewissoftwire 2 years ago
parent
commit
929d197b41
  1. 87
      Dockerfile
  2. 41
      Dockerfile_dev
  3. 346
      app/constants/location_constants.rb
  4. 2
      app/services/imports/scheme_location_import_service.rb
  5. 6
      docker-compose.yml
  6. 2
      docs/Gemfile.lock
  7. 2
      spec/helpers/filters_helper_spec.rb
  8. 9
      spec/services/csv/lettings_log_csv_service_spec.rb
  9. 13
      spec/services/imports/scheme_location_import_service_spec.rb
  10. 6
      yarn.lock

87
Dockerfile

@ -1,71 +1,72 @@
# Build compilation image FROM ruby:3.1.4-alpine3.18 as base
FROM ruby:3.1.4-alpine as builder
# The application runs from /app
WORKDIR /app WORKDIR /app
# Add the timezone as it's not configured by default in Alpine # Add the timezone as it's not configured by default in Alpine
RUN apk add --update --no-cache tzdata && \ RUN apk add --update --no-cache tzdata && \
cp /usr/share/zoneinfo/Europe/London /etc/localtime && \ cp /usr/share/zoneinfo/Europe/London /etc/localtime && \
echo "Europe/London" > /etc/timezone echo "Europe/London" > /etc/timezone
# build-base: complication tools for bundle # build-base: compilation tools for bundle
# yarn: node package manager # yarn: node package manager
# postgresql-dev: postgres driver and libraries # postgresql-dev: postgres driver and libraries
RUN apk add --no-cache build-base yarn postgresql-dev git RUN apk add --no-cache build-base=0.5-r3 yarn=1.22.19-r0 postgresql13-dev=13.11-r0 git=2.40.1-r0 bash=5.2.15-r5
# Install bundler to run bundle exec # Bundler version should be the same version as what the Gemfile.lock was bundled with
# This should be the same version as the Gemfile.lock
RUN gem install bundler:2.3.14 --no-document RUN gem install bundler:2.3.14 --no-document
# Install gems defined in Gemfile
COPY .ruby-version Gemfile Gemfile.lock /app/ COPY .ruby-version Gemfile Gemfile.lock /app/
RUN bundle config set without "development test" RUN bundle config set without "development test"
ARG BUNDLE_FLAGS="--jobs=4 --no-binstubs --no-cache" RUN bundle install --jobs=4 --no-binstubs --no-cache
RUN bundle install ${BUNDLE_FLAGS}
# Install node packages defined in package.json, including webpack
COPY package.json yarn.lock /app/ COPY package.json yarn.lock /app/
RUN yarn install --frozen-lockfile RUN yarn install --frozen-lockfile
# Copy all files to /app (except what is defined in .dockerignore)
COPY . /app/ COPY . /app/
# Compile assets and run webpack. We set a dummy secret key. RUN bundle exec rake assets:precompile
RUN RAILS_ENV=production bundle exec rails SECRET_KEY_BASE=pickasecuretoken assets:precompile
# Cleanup to save space in the production image ENV PORT=8080
RUN rm -rf node_modules log tmp && \ EXPOSE ${PORT}
rm -rf /usr/local/bundle/cache && \
rm -rf .env && \
find /usr/local/bundle/gems -name "*.c" -delete && \
find /usr/local/bundle/gems -name "*.h" -delete && \
find /usr/local/bundle/gems -name "*.o" -delete && \
find /usr/local/bundle/gems -name "*.html" -delete
# Build runtime image RUN adduser --system --no-create-home nonroot
FROM ruby:3.1.4-alpine as production
# The application runs from /app FROM base as test
WORKDIR /app
# Add postgres driver library RUN bundle config set without ""
# Add the timezone as it's not configured by default in Alpine RUN bundle install --jobs=4 --no-binstubs --no-cache
RUN apk add --update --no-cache libpq tzdata && \
cp /usr/share/zoneinfo/Europe/London /etc/localtime && \
echo "Europe/London" > /etc/timezone
# Copy files generated in the builder image # Install gecko driver for Capybara tests
COPY --from=builder /app /app RUN apk add firefox
COPY --from=builder /usr/local/bundle/ /usr/local/bundle/ RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.31.0/geckodriver-v0.31.0-linux64.tar.gz \
&& tar -xvzf geckodriver-v0.31.0-linux64.tar.gz \
&& rm geckodriver-v0.31.0-linux64.tar.gz \
&& chmod +x geckodriver \
&& mv geckodriver /usr/local/bin/
ARG GIT_COMMIT_SHA="UNKNOWN" CMD bundle exec rake parallel:setup && bundle exec rake parallel:spec
ENV GIT_COMMIT_SHA=${GIT_COMMIT_SHA}
RUN echo ${GIT_COMMIT_SHA} > ./GIT_COMMIT_SHA
ENV PORT=8080 FROM base as development
EXPOSE ${PORT} # We expect the rake assets:precompile command to create these directories, but mkdir -p will create them if they don't already exist
RUN mkdir -p tmp log
RUN chown -R nonroot tmp log
RUN chown nonroot db/schema.rb
RUN bundle config set without "test"
RUN bundle install --jobs=4 --no-binstubs --no-cache
USER nonroot
CMD bundle exec rails s -e ${RAILS_ENV} -p ${PORT} --binding=0.0.0.0
FROM base as production
# We expect the rake assets:precompile command to create these directories, but mkdir -p will create them if they don't already exist
RUN mkdir -p tmp log
RUN chown -R nonroot tmp log
RUN chown nonroot db/schema.rb
USER nonroot
CMD RAILS_ENV=${RAILS_ENV} bundle exec rake db:migrate && bundle exec rails s -e ${RAILS_ENV} -p ${PORT} --binding=0.0.0.0 CMD bundle exec rails s -e ${RAILS_ENV} -p ${PORT} --binding=0.0.0.0

41
Dockerfile_dev

@ -1,41 +0,0 @@
# Build compilation image
FROM ruby:3.1.4-alpine
# The application runs from /app
WORKDIR /app
# Add the timezone as it's not configured by default in Alpine
RUN apk add --update --no-cache tzdata && cp /usr/share/zoneinfo/Europe/London /etc/localtime && echo "Europe/London" > /etc/timezone
RUN apk add --no-cache build-base yarn postgresql-dev git bash
# Install bundler to run bundle exec
# This should be the same version as the Gemfile.lock
RUN gem install bundler:2.3.14 --no-document
# Install gems defined in Gemfile
COPY .ruby-version Gemfile Gemfile.lock /app/
ARG BUNDLE_FLAGS="--jobs=4 --no-binstubs --no-cache"
RUN bundle install ${BUNDLE_FLAGS}
# Install node packages defined in package.json, including webpack
COPY package.json yarn.lock /app/
RUN yarn install --frozen-lockfile
# Install gecko driver for Capybara tests
RUN apk add firefox
RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.31.0/geckodriver-v0.31.0-linux64.tar.gz \
&& tar -xvzf geckodriver-v0.31.0-linux64.tar.gz \
&& rm geckodriver-v0.31.0-linux64.tar.gz \
&& chmod +x geckodriver \
&& mv geckodriver /usr/local/bin/
# Copy all files to /app (except what is defined in .dockerignore)
COPY . /app/
ENV PORT=8080
EXPOSE ${PORT}
CMD RAILS_ENV=${RAILS_ENV} bundle exec rake db:migrate && bundle exec rails s -e ${RAILS_ENV} -p ${PORT} --binding=0.0.0.0

346
app/constants/location_constants.rb

@ -0,0 +1,346 @@
module LocationConstants
LA_NAME_TO_CODE = {
"102" => "E07000223",
"104" => "E07000026",
"107" => "E07000032",
"110" => "E07000224",
"111" => "E07000170",
"112" => "E07000105",
"113" => "E07000004",
"114" => "E07000200",
"115" => "E09000002",
"116" => "E09000003",
"117" => "E08000016",
"118" => "E07000027",
"119" => "E07000066",
"120" => "E07000084",
"121" => "E07000171",
"127" => "E09000004",
"128" => "E08000025",
"129" => "E07000129",
"130" => "E06000008",
"131" => "E06000009",
"134" => "E07000033",
"135" => "E08000001",
"137" => "E07000136",
"138" => "E06000028",
"139" => "E06000036",
"140" => "E08000032",
"141" => "E07000067",
"142" => "E07000143",
"144" => "E09000005",
"145" => "E07000068",
"148" => "E06000023",
"149" => "E07000144",
"150" => "E09000006",
"151" => "E07000234",
"152" => "E07000095",
"153" => "E07000172",
"154" => "E07000117",
"155" => "E08000002",
"156" => "E08000033",
"157" => "E07000008",
"158" => "E09000007",
"159" => "E07000192",
"160" => "E07000106",
"163" => "E07000028",
"167" => "E07000069",
"169" => "E07000130",
"170" => "E07000070",
"171" => "E07000078",
"172" => "E07000177",
"175" => "E07000034",
"176" => "E07000225",
"177" => "E07000005",
"178" => "E07000118",
"179" => "E07000048",
"181" => "E07000071",
"184" => "E07000029",
"185" => "E07000150",
"186" => "E07000079",
"187" => "E08000026",
"188" => "E07000163",
"189" => "E07000226",
"191" => "E09000008",
"193" => "E07000096",
"194" => "E06000005",
"195" => "E07000107",
"196" => "E07000151",
"198" => "E06000015",
"201" => "E08000017",
"202" => "E07000108",
"203" => "E08000027",
"206" => "E09000009",
"208" => "E07000009",
"209" => "E07000040",
"210" => "E07000085",
"211" => "E07000242",
"212" => "E07000137",
"213" => "E07000152",
"214" => "E07000193",
"215" => "E07000061",
"216" => "E07000086",
"217" => "E07000030",
"219" => "E07000207",
"220" => "E09000010",
"221" => "E07000072",
"222" => "E07000208",
"223" => "E07000036",
"224" => "E07000041",
"225" => "E07000087",
"226" => "E07000010",
"227" => "E07000201",
"228" => "E07000080",
"229" => "E07000119",
"230" => "E08000037",
"231" => "E07000173",
"234" => "E07000081",
"236" => "E07000088",
"237" => "E07000109",
"238" => "E07000145",
"239" => "E09000011",
"241" => "E07000209",
"242" => "E09000012",
"243" => "E06000006",
"244" => "E07000164",
"245" => "E09000013",
"246" => "E07000131",
"247" => "E09000014",
"248" => "E07000073",
"249" => "E07000165",
"250" => "E09000015",
"251" => "E07000089",
"252" => "E06000001",
"253" => "E07000062",
"254" => "E07000090",
"255" => "E09000016",
"257" => "E07000098",
"258" => "E07000037",
"259" => "E09000017",
"260" => "E07000132",
"262" => "E07000227",
"263" => "E09000018",
"265" => "E07000011",
"266" => "E07000120",
"267" => "E07000202",
"268" => "E09000019",
"271" => "E09000020",
"273" => "E07000153",
"274" => "E06000010",
"275" => "E09000021",
"277" => "E08000034",
"278" => "E08000011",
"279" => "E09000022",
"280" => "E07000121",
"281" => "E06000003",
"282" => "E08000035",
"283" => "E06000016",
"285" => "E07000063",
"286" => "E09000023",
"287" => "E07000194",
"288" => "E07000138",
"289" => "E08000012",
"292" => "E06000032",
"294" => "E07000110",
"295" => "E07000074",
"296" => "E07000235",
"297" => "E08000003",
"298" => "E07000174",
"302" => "E07000133",
"303" => "E07000187",
"305" => "E09000024",
"307" => "E07000203",
"308" => "E07000228",
"309" => "E06000002",
"310" => "E06000042",
"311" => "E07000210",
"315" => "E07000091",
"316" => "E07000175",
"317" => "E06000037",
"318" => "E08000021",
"319" => "E07000195",
"320" => "E09000025",
"323" => "E07000043",
"324" => "E07000050",
"325" => "E07000038",
"326" => "E07000099",
"327" => "E07000139",
"328" => "E07000147",
"329" => "E06000061",
"330" => "E08000022",
"331" => "E07000218",
"332" => "E07000134",
"335" => "E07000154",
"337" => "E07000148",
"338" => "E06000018",
"339" => "E07000219",
"340" => "E07000135",
"342" => "E08000004",
"344" => "E07000178",
"345" => "E07000122",
"347" => "E06000031",
"348" => "E06000026",
"349" => "E06000029",
"350" => "E06000044",
"352" => "E07000123",
"353" => "E07000051",
"355" => "E06000038",
"356" => "E09000026",
"357" => "E07000236",
"358" => "E07000211",
"363" => "E07000124",
"364" => "E09000027",
"365" => "E07000166",
"366" => "E08000005",
"367" => "E07000075",
"368" => "E07000125",
"369" => "E07000064",
"370" => "E08000018",
"371" => "E07000220",
"372" => "E07000212",
"373" => "E07000176",
"374" => "E07000092",
"375" => "E06000017",
"376" => "E07000167",
"377" => "E08000006",
"379" => "E08000028",
"380" => "E07000168",
"383" => "E07000188",
"384" => "E08000014",
"385" => "E07000169",
"386" => "E07000111",
"387" => "E08000019",
"388" => "E07000112",
"390" => "E06000039",
"391" => "E08000029",
"393" => "E07000012",
"394" => "E07000039",
"395" => "E07000044",
"397" => "E07000140",
"398" => "E07000141",
"399" => "E07000031",
"400" => "E07000149",
"401" => "E07000155",
"402" => "E07000179",
"404" => "E07000126",
"406" => "E07000196",
"407" => "E08000023",
"409" => "E06000045",
"410" => "E06000033",
"411" => "E09000028",
"412" => "E07000213",
"413" => "E07000240",
"414" => "E07000204",
"415" => "E08000013",
"416" => "E07000197",
"417" => "E07000198",
"418" => "E07000243",
"419" => "E08000007",
"420" => "E06000004",
"421" => "E06000021",
"422" => "E07000221",
"423" => "E07000082",
"424" => "E07000205",
"425" => "E08000024",
"426" => "E07000214",
"427" => "E09000029",
"428" => "E07000113",
"431" => "E08000008",
"432" => "E07000199",
"433" => "E07000215",
"434" => "E07000190",
"436" => "E07000045",
"437" => "E07000076",
"438" => "E07000093",
"439" => "E07000083",
"440" => "E06000030",
"441" => "E07000114",
"442" => "E06000020",
"443" => "E07000102",
"444" => "E06000034",
"445" => "E07000042",
"446" => "E07000115",
"447" => "E06000027",
"449" => "E07000046",
"450" => "E09000030",
"451" => "E08000009",
"452" => "E07000116",
"454" => "E07000077",
"456" => "E07000180",
"458" => "E08000036",
"459" => "E08000030",
"460" => "E09000031",
"461" => "E09000032",
"464" => "E06000007",
"465" => "E07000222",
"466" => "E07000103",
"467" => "E07000206",
"468" => "E07000216",
"469" => "E07000065",
"471" => "E07000156",
"472" => "E07000241",
"473" => "E07000035",
"474" => "E07000047",
"475" => "E07000052",
"476" => "E07000127",
"477" => "E07000142",
"478" => "E07000146",
"479" => "E07000181",
"480" => "E07000191",
"481" => "E06000062",
"482" => "E09000033",
"483" => "E07000053",
"484" => "E08000010",
"485" => "E07000049",
"486" => "E07000094",
"487" => "E06000040",
"488" => "E08000015",
"489" => "E07000217",
"490" => "E06000041",
"491" => "E08000031",
"492" => "E06000024",
"493" => "E07000237",
"494" => "E07000229",
"496" => "E07000238",
"497" => "E07000007",
"498" => "E07000128",
"499" => "E07000239",
"500" => "E07000189",
"501" => "E06000014",
"502" => "E07000006",
"503" => "E09000001",
"504" => "E06000053",
"505" => "E06000046",
"506" => "E06000025",
"507" => "E06000022",
"508" => "E06000012",
"509" => "E06000013",
"510" => "E06000011",
"511" => "E06000043",
"512" => "E06000019",
"513" => "E06000035",
"514" => "E06000063",
"515" => "E06000064",
"516" => "E06000065",
"517" => "E06000066",
"601" => "E06000055",
"602" => "E06000056",
"603" => "E06000050",
"604" => "E06000052",
"605" => "E06000047",
"606" => "E06000049",
"607" => "E06000057",
"608" => "E06000051",
"609" => "E06000054",
"700" => "E06000058",
"701" => "E06000059",
"702" => "E07000244",
"703" => "E07000246",
"704" => "E07000245",
"990" => "S92000003",
"991" => "W92000004",
"992" => "N92000002",
"993" => "9300000XX",
"999" => "E06000060",
}.freeze
end

2
app/services/imports/scheme_location_import_service.rb

@ -93,6 +93,7 @@ module Imports
attributes["location_old_id"] = string_or_nil(xml_doc, "id") attributes["location_old_id"] = string_or_nil(xml_doc, "id")
attributes["location_old_visible_id"] = string_or_nil(xml_doc, "visible-id") attributes["location_old_visible_id"] = string_or_nil(xml_doc, "visible-id")
attributes["scheme_old_id"] = string_or_nil(xml_doc, "mgmtgroup") attributes["scheme_old_id"] = string_or_nil(xml_doc, "mgmtgroup")
attributes["location_code"] = LocationConstants::LA_NAME_TO_CODE[string_or_nil(xml_doc, "local-authority-name")]
attributes attributes
end end
@ -106,6 +107,7 @@ module Imports
old_visible_id: attributes["location_old_visible_id"], old_visible_id: attributes["location_old_visible_id"],
old_id: attributes["location_old_id"], old_id: attributes["location_old_id"],
startdate: attributes["start_date"], startdate: attributes["start_date"],
location_code: attributes["location_code"],
scheme:, scheme:,
) )
if attributes["end_date"] if attributes["end_date"]

6
docker-compose.yml

@ -11,7 +11,7 @@ services:
volumes: volumes:
- dbdata:/var/lib/postgresql/data - dbdata:/var/lib/postgresql/data
ports: ports:
- 5433:5432 # Mapped to 5433 in case Postgres is already running locally on 5432 - 8081:5432 # Mapped to 8081 in case Postgres is already running locally on 5432
environment: environment:
- POSTGRES_PASSWORD=password - POSTGRES_PASSWORD=password
- POSTGRES_USER=postgres - POSTGRES_USER=postgres
@ -20,7 +20,9 @@ services:
app: app:
build: build:
context: . context: .
dockerfile: ./Dockerfile_dev dockerfile: ./Dockerfile
target: development
platform: linux/amd64
ports: ports:
- 8080:8080 - 8080:8080
depends_on: depends_on:

2
docs/Gemfile.lock

@ -13,7 +13,7 @@ GEM
execjs execjs
coffee-script-source (1.11.1) coffee-script-source (1.11.1)
colorator (1.1.0) colorator (1.1.0)
commonmarker (0.23.9) commonmarker (0.23.10)
concurrent-ruby (1.2.2) concurrent-ruby (1.2.2)
dnsruby (1.61.9) dnsruby (1.61.9)
simpleidn (~> 0.1) simpleidn (~> 0.1)

2
spec/helpers/filters_helper_spec.rb

@ -176,7 +176,7 @@ RSpec.describe FiltersHelper do
let(:user) { FactoryBot.create(:user, :support, organisation: child_organisation) } let(:user) { FactoryBot.create(:user, :support, organisation: child_organisation) }
it "returns a list of all organisations" do it "returns a list of all organisations" do
expect(owning_organisation_filter_options(user)).to eq([ expect(owning_organisation_filter_options(user)).to match_array([
OpenStruct.new(id: "", name: "Select an option"), OpenStruct.new(id: "", name: "Select an option"),
OpenStruct.new(id: child_organisation.id, name: "Child organisation"), OpenStruct.new(id: child_organisation.id, name: "Child organisation"),
OpenStruct.new(id: absorbed_organisation.id, name: "Absorbed organisation"), OpenStruct.new(id: absorbed_organisation.id, name: "Absorbed organisation"),

9
spec/services/csv/lettings_log_csv_service_spec.rb

@ -28,6 +28,15 @@ RSpec.describe Csv::LettingsLogCsvService do
let(:logs) { [log] } let(:logs) { [log] }
let(:headers) { csv.first } let(:headers) { csv.first }
before do
Timecop.freeze(fixed_time)
Singleton.__init__(FormHandler)
end
after do
Timecop.return
end
it "calls the form handler to get all questions in order when initialized" do it "calls the form handler to get all questions in order when initialized" do
allow(FormHandler).to receive(:instance).and_return(form_handler_mock) allow(FormHandler).to receive(:instance).and_return(form_handler_mock)
allow(form_handler_mock).to receive(:ordered_lettings_questions_for_all_years).and_return([]) allow(form_handler_mock).to receive(:ordered_lettings_questions_for_all_years).and_return([])

13
spec/services/imports/scheme_location_import_service_spec.rb

@ -17,7 +17,7 @@ RSpec.describe Imports::SchemeLocationImportService do
end end
before do before do
WebMock.stub_request(:get, /api.postcodes.io\/postcodes/) WebMock.stub_request(:get, /api\.postcodes\.io\/postcodes\/S446EJ/)
.to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {}) .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
end end
@ -143,6 +143,7 @@ RSpec.describe Imports::SchemeLocationImportService do
location = location_service.create_scheme_location(location_xml) location = location_service.create_scheme_location(location_xml)
expect(location.name).to eq("Location 1") expect(location.name).to eq("Location 1")
expect(location.postcode).to eq("S44 6EJ") expect(location.postcode).to eq("S44 6EJ")
expect(location.location_code).to eq("E08000035")
expect(location.units).to eq(5) expect(location.units).to eq(5)
expect(location.mobility_type).to eq("Fitted with equipment and adaptations") expect(location.mobility_type).to eq("Fitted with equipment and adaptations")
expect(location.type_of_unit).to eq("Bungalow") expect(location.type_of_unit).to eq("Bungalow")
@ -226,5 +227,15 @@ RSpec.describe Imports::SchemeLocationImportService do
expect(location.scheme.support_type).to eq(nil) expect(location.scheme.support_type).to eq(nil)
end end
end end
context "and postcode does not return a location code" do
before { location_xml.at_xpath("//scheme:postcode").content = "A1 1AA" }
it "imports location code correctly" do
location = location_service.create_scheme_location(location_xml)
expect(location.postcode).to eq("A1 1AA")
expect(location.location_code).to eq("E07000033")
end
end
end end
end end

6
yarn.lock

@ -1843,9 +1843,9 @@ camelcase@^6.3.0:
integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==
caniuse-lite@^1.0.30001400: caniuse-lite@^1.0.30001400:
version "1.0.30001410" version "1.0.30001519"
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001410.tgz#b5a86366fbbf439d75dd3db1d21137a73e829f44" resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001519.tgz"
integrity sha512-QoblBnuE+rG0lc3Ur9ltP5q47lbguipa/ncNMyyGuqPk44FxbScWAeEO+k5fSQ8WekdAK4mWqNs1rADDAiN5xQ== integrity sha512-0QHgqR+Jv4bxHMp8kZ1Kn8CH55OikjKJ6JmKkZYP1F3D7w+lnFXF70nG5eNfsZS89jadi5Ywy5UCSKLAglIRkg==
caseless@^0.12.0, caseless@~0.12.0: caseless@^0.12.0, caseless@~0.12.0:
version "0.12.0" version "0.12.0"

Loading…
Cancel
Save