diff --git a/.dockerdev/.bashrc b/.dockerdev/.bashrc new file mode 100644 index 0000000..1e8f702 --- /dev/null +++ b/.dockerdev/.bashrc @@ -0,0 +1,2 @@ +alias be="bundle exec" +PS1='🐳\[\033[01;31m\] ❯\[\033[00m\] ' diff --git a/.dockerdev/.psqlrc b/.dockerdev/.psqlrc new file mode 100644 index 0000000..05c0dcb --- /dev/null +++ b/.dockerdev/.psqlrc @@ -0,0 +1,26 @@ +-- Don't display the "helpful" message on startup. +\set QUIET 1 + +-- Allow specifying the path to history file via `PSQL_HISTFILE` env variable +-- (and fallback to the default $HOME/.psql_history otherwise) +\set HISTFILE `[ -z $PSQL_HISTFILE ] && echo $HOME/.psql_history || echo $PSQL_HISTFILE` + +-- Show how long each query takes to execute +\timing + +-- Use best available output format +\x auto + +-- Verbose error reports +\set VERBOSITY verbose + +-- If a command is run more than once in a row, +-- only store it once in the history +\set HISTCONTROL ignoredups +\set COMP_KEYWORD_CASE upper + +-- By default, NULL displays as an empty space. Is it actually an empty +-- string, or is it null? This makes that distinction visible +\pset null '[NULL]' + +\unset QUIET diff --git a/.dockerdev/Aptfile b/.dockerdev/Aptfile new file mode 100644 index 0000000..f2f1f7b --- /dev/null +++ b/.dockerdev/Aptfile @@ -0,0 +1,3 @@ +vim +file +zip diff --git a/.dockerdev/Dockerfile b/.dockerdev/Dockerfile new file mode 100644 index 0000000..14d1954 --- /dev/null +++ b/.dockerdev/Dockerfile @@ -0,0 +1,64 @@ +ARG RUBY_VERSION=3.3.0 +ARG DISTRO_NAME=bookworm + +FROM ruby:$RUBY_VERSION-slim-$DISTRO_NAME + +# Common dependencies +RUN apt-get update -qq \ + && DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + build-essential \ + gnupg2 \ + curl \ + less \ + git \ + && apt-get clean \ + && rm -rf /var/cache/apt/archives/* \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && truncate -s 0 /var/log/*log + +ARG PG_MAJOR +ARG DISTRO_NAME +RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgres-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/postgres-archive-keyring.gpg] https://apt.postgresql.org/pub/repos/apt/" \ + $DISTRO_NAME-pgdg main $PG_MAJOR | tee /etc/apt/sources.list.d/postgres.list > /dev/null +RUN apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + libpq-dev \ + postgresql-client-$PG_MAJOR \ + && apt-get clean \ + && rm -rf /var/cache/apt/archives/* \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && truncate -s 0 /var/log/*log + +# Application dependencies +COPY Aptfile /tmp/Aptfile +RUN apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + $(grep -Ev '^\s*#' /tmp/Aptfile | xargs) \ + && apt-get clean \ + && rm -rf /var/cache/apt/archives/* \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && truncate -s 0 /var/log/*log + +# Configure bundler +ENV LANG=C.UTF-8 \ + BUNDLE_JOBS=4 \ + BUNDLE_RETRY=3 + +# Store Bundler settings in the project's root +ENV BUNDLE_APP_CONFIG=.bundle + +# Upgrade RubyGems and install the latest Bundler version +RUN gem update --system && \ + gem install bundler + +# Create a directory for the app code +WORKDIR /rails + +# Use Bash as the default command +# Entrypoint prepares the database. +ENTRYPOINT ["/rails/bin/docker-entrypoint"] + +# Start the server by default, this can be overwritten at runtime +EXPOSE 3000 +CMD ["./bin/rails", "server"] diff --git a/.dockerdev/compose.yml b/.dockerdev/compose.yml new file mode 100644 index 0000000..58e8032 --- /dev/null +++ b/.dockerdev/compose.yml @@ -0,0 +1,102 @@ +x-app: + &app + build: + context: . + args: + RUBY_VERSION: '3.3.0' + PG_MAJOR: '17' + image: library/dev:1.0.0 + environment: + &env + RAILS_ENV: ${RAILS_ENV:-development} + env_file: + - ./.env + tmpfs: + - /tmp + - /app/tmp/pids + +x-backend: + &backend + <<: *app + stdin_open: true + tty: true + volumes: + - ..:/rails:cached + - bundle:/usr/local/bundle + - rails_cache:/rails/tmp/cache + - history:/usr/local/hist + - ./.psqlrc:/root/.psqlrc:ro + - ./.bashrc:/root/.bashrc:ro + - "$DOCKER_COMPOSE_ARCHIVE_FOLDER:/rails/db/data" + environment: + &backend_environment + <<: *env + DATABASE_URL: postgres://postgres:postgres@postgres:5432 + MALLOC_ARENA_MAX: 2 + WEB_CONCURRENCY: ${WEB_CONCURRENCY:-1} + BOOTSNAP_CACHE_DIR: /usr/local/bundle/_bootsnap + XDG_DATA_HOME: /rails/tmp/cache + HISTFILE: /usr/local/hist/.bash_history + PSQL_HISTFILE: /usr/local/hist/.psql_history + IRB_HISTFILE: /usr/local/hist/.irb_history + EDITOR: vi + RAILS_LOG_TO_STDOUT: 'yes' + PAGER: 'more' + depends_on: + postgres: + condition: service_healthy + mongodb: + condition: service_healthy + + +services: + runner: + <<: *backend + command: /bin/bash + + rails: + <<: *backend + command: bundle exec rails + + web: + <<: *backend + command: > + sh -c "rm -f tmp/pids/server.pid && + bundle exec rails s -p 3000 -b '0.0.0.0'" + ports: + - '3000:3000' + + postgres: + image: postgres:17 + volumes: + - .psqlrc:/root/.psqlrc:ro + - postgres:/var/lib/postgresql/data + - history:/usr/local/hist + environment: + PSQL_HISTFILE: /usr/local/hist/.psql_history + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + healthcheck: + test: pg_isready -U postgres -h 127.0.0.1 + interval: 5s + + mongodb: + image: mongo:latest + ports: + - '27017:27017' + volumes: + - mongodb:/data/db + healthcheck: + test: ["CMD","mongosh", "--eval", "db.adminCommand('ping')"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 5s + +volumes: + bundle: + history: + rails_cache: + postgres: + mongodb: diff --git a/Gemfile b/Gemfile index fc76bca..c78d10d 100644 --- a/Gemfile +++ b/Gemfile @@ -5,6 +5,9 @@ gem 'puma', '>= 6.5.0' # Базы данных gem 'pg' +gem 'mongoid', '~> 9.0' +gem 'bson_ext' +gem 'kaminari-mongoid' # Многопоточное выполнение gem 'parallel' @@ -29,6 +32,9 @@ gem 'activeadmin' gem 'activeadmin_addons' gem 'devise' +gem 'alba' +gem 'kaminari' + group :development, :test do gem 'bundler-audit' gem 'capybara' diff --git a/Gemfile.lock b/Gemfile.lock index 63e5241..d96ba5a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -95,6 +95,8 @@ GEM uri (>= 0.13.1) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) + alba (3.5.0) + ostruct (~> 0.6) arbre (1.7.0) activesupport (>= 3.0.0) ruby2_keywords (>= 0.0.2) @@ -104,6 +106,8 @@ GEM benchmark (0.4.0) bigdecimal (3.1.8) bindex (0.8.1) + bson (5.0.2) + bson_ext (1.5.1) builder (3.3.0) bundler-audit (0.9.2) bundler (>= 1.2.0, < 3) @@ -210,6 +214,9 @@ GEM activerecord kaminari-core (= 1.2.2) kaminari-core (1.2.2) + kaminari-mongoid (1.0.2) + kaminari-core (~> 1.0) + mongoid language_server-protocol (3.17.0.3) logger (1.6.0) loofah (2.23.1) @@ -226,6 +233,12 @@ GEM mini_mime (1.1.5) mini_portile2 (2.8.8) minitest (5.25.1) + mongo (2.21.0) + bson (>= 4.14.1, < 6.0.0) + mongoid (9.0.6) + activemodel (>= 5.1, < 8.1, != 7.0.0) + concurrent-ruby (>= 1.0.5, < 2.0) + mongo (>= 2.18.0, < 3.0.0) net-imap (0.5.6) date net-protocol @@ -448,6 +461,8 @@ DEPENDENCIES activeadmin activeadmin_addons activerecord-import + alba + bson_ext bundler-audit capybara config @@ -460,6 +475,9 @@ DEPENDENCIES factory_bot_rails fasterer ffaker + kaminari + kaminari-mongoid + mongoid (~> 9.0) parallel pg pry-byebug diff --git a/app/controllers/books_controller.rb b/app/controllers/books_controller.rb new file mode 100644 index 0000000..a4e2710 --- /dev/null +++ b/app/controllers/books_controller.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class BooksController < ApplicationController + def index + books = Mongo::Book.page(page).per(Settings.app.items_per_page) + render json: BookSerializer.new(books).serialize + end + + private + + def page + [ params[:page].to_i, 1 ].max + end +end diff --git a/app/models/mongo/book.rb b/app/models/mongo/book.rb new file mode 100644 index 0000000..02218c0 --- /dev/null +++ b/app/models/mongo/book.rb @@ -0,0 +1,28 @@ +module Mongo + class Book + include Mongoid::Document + include Mongoid::Timestamps + + field :title, type: String + field :series, type: String + field :serno, type: String + field :libid, type: Integer + field :size, type: Integer + field :filename, type: Integer + field :del, type: Boolean + field :ext, type: String + field :published_at, type: Date + field :insno, type: String + + has_and_belongs_to_many :authors + has_and_belongs_to_many :genres + has_and_belongs_to_many :keywords + + belongs_to :folder, optional: true, class_name: 'Mongo::Folder' + belongs_to :language, optional: true, class_name: 'Mongo::Language' + + PUBLIC_FIELDS = %w[del ext filename folder_id insno language_id + libid series serno size title published_at + updated_at created_at].freeze + end +end diff --git a/app/models/mongo/folder.rb b/app/models/mongo/folder.rb new file mode 100644 index 0000000..392600f --- /dev/null +++ b/app/models/mongo/folder.rb @@ -0,0 +1,9 @@ +module Mongo + class Folder + include Mongoid::Document + include Mongoid::Timestamps + + validates :name, presence: true + validates :name, uniqueness: true + end +end diff --git a/app/models/mongo/genre_group.rb b/app/models/mongo/genre_group.rb new file mode 100644 index 0000000..d312091 --- /dev/null +++ b/app/models/mongo/genre_group.rb @@ -0,0 +1,6 @@ +module Mongo + class GenreGroup + include Mongoid::Document + include Mongoid::Timestamps + end +end diff --git a/app/models/mongo/language.rb b/app/models/mongo/language.rb new file mode 100644 index 0000000..9060a16 --- /dev/null +++ b/app/models/mongo/language.rb @@ -0,0 +1,9 @@ +module Mongo + class Language + include Mongoid::Document + include Mongoid::Timestamps + + validates :slug, presence: true + validates :slug, uniqueness: true + end +end diff --git a/app/serializers/book_serializer.rb b/app/serializers/book_serializer.rb new file mode 100644 index 0000000..59466d3 --- /dev/null +++ b/app/serializers/book_serializer.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class BookSerializer + include ::Alba::Resource + + attributes :id, :title, :created_at, :updated_at, :series, :serno, :libid, :size, :filename, + :del, :ext, :published_at, :insno + + attribute :folder do |book| + book.folder.try(:name) if book.folder + end +end diff --git a/app/services/mongo/books/parse.rb b/app/services/mongo/books/parse.rb new file mode 100644 index 0000000..3f6b518 --- /dev/null +++ b/app/services/mongo/books/parse.rb @@ -0,0 +1,47 @@ +class Mongo::Books::Parse + include Callable + extend Dry::Initializer + + option :books, type: Dry::Types['strict.array'] + + attr_reader :authors, :keywords, :folders, :genres, :languages + + def call + read_file + + true + end + + def book_attribures(attr, folder_id, language_id) + { + title: attr[2], + series: attr[3], + serno: attr[4], + libid: attr[5].to_i, + size: attr[6].to_i, + filename: attr[7].to_i, + del: attr[8] == '1', + ext: attr[9], + published_at: attr[10], + insno: attr[11], + folder_id: folder_id, + language_id: language_id + } + end + + def folders_map + @folders ||= Mongo::Folder.pluck(:name, :id).to_h + end + + def languages_map + @languages ||= Mongo::Language.pluck(:slug, :id).to_h + end + + def read_file + attribures = books.each_with_object([]) do |line, association| + arr = line.split(4.chr) + association << book_attribures(arr, folders_map[arr[12]], languages_map[arr[13]]) + end + Mongo::Book.collection.insert_many(attribures) + end +end diff --git a/app/services/seeds/mongo/genre_load.rb b/app/services/seeds/mongo/genre_load.rb new file mode 100644 index 0000000..19ccf82 --- /dev/null +++ b/app/services/seeds/mongo/genre_load.rb @@ -0,0 +1,16 @@ +class Seeds::Mongo::GenreLoad + include Callable + extend Dry::Initializer + + option :filename, type: Dry::Types['strict.string'] + + def call + Mongo::GenreGroup.delete_all + Mongo::GenreGroup.collection.insert_many seeds_from_yaml(filename) + end + + + def seeds_from_yaml(file) + YAML.load_file(Rails.root.join(file))['items'] || [] + end +end diff --git a/app/services/seeds/mongo/language_load.rb b/app/services/seeds/mongo/language_load.rb new file mode 100644 index 0000000..f4a592d --- /dev/null +++ b/app/services/seeds/mongo/language_load.rb @@ -0,0 +1,15 @@ +class Seeds::Mongo::LanguageLoad + include Callable + extend Dry::Initializer + + option :filename, type: Dry::Types['strict.string'] + + def call + Mongo::Language.delete_all + Mongo::Language.collection.insert_many seeds_from_yaml(filename) + end + + def seeds_from_yaml(file) + YAML.load_file(Rails.root.join(file))['items'] || [] + end +end diff --git a/config/initializers/mongoid.rb b/config/initializers/mongoid.rb new file mode 100644 index 0000000..7ceeac9 --- /dev/null +++ b/config/initializers/mongoid.rb @@ -0,0 +1,25 @@ +# rubocop:todo all +Mongoid.configure do + target_version = "9.0" + + # Load Mongoid behavior defaults. This automatically sets + # features flags (refer to documentation) + config.load_defaults target_version + + # It is recommended to use config/mongoid.yml for most Mongoid-related + # configuration, whenever possible, but if you prefer, you can set + # configuration values here, instead: + # + # config.log_level = :debug + # + # Note that the settings in config/mongoid.yml always take precedence, + # whatever else is set here. +end + +# Enable Mongo driver query cache for Rack +# Rails.application.config.middleware.use(Mongo::QueryCache::Middleware) + +# Enable Mongo driver query cache for ActiveJob +# ActiveSupport.on_load(:active_job) do +# include Mongo::QueryCache::Middleware::ActiveJob +# end diff --git a/config/mongoid.yml b/config/mongoid.yml new file mode 100644 index 0000000..8ad2b50 --- /dev/null +++ b/config/mongoid.yml @@ -0,0 +1,298 @@ +development: + # Configure available database clients. (required) + clients: + # Defines the default client. (required) + default: + # Mongoid can connect to a URI accepted by the driver: + # uri: mongodb://user:password@mongodb.domain.com:27017/library_development + + # Otherwise define the parameters separately. + # This defines the name of the default database that Mongoid can connect to. + # (required). + database: library_development + # Provides the hosts the default client can connect to. Must be an array + # of host:port pairs. (required) + hosts: + - mongodb:27017 + options: + # Note that all options listed below are Ruby driver client options (the mongo gem). + # Please refer to the driver documentation of the version of the mongo gem you are using + # for the most up-to-date list of options. + + # Change the default write concern. (default = { w: 1 }) + # write: + # w: 1 + + # Change the default read preference. Valid options for mode are: :secondary, + # :secondary_preferred, :primary, :primary_preferred, :nearest + # (default: primary) + # read: + # mode: :secondary_preferred + # tag_sets: + # - use: web + + # The name of the user for authentication. + # user: 'user' + + # The password of the user for authentication. + # password: 'password' + + # The user's database roles. + # roles: + # - 'dbOwner' + + # Change the default authentication mechanism. Valid options include: + # :scram, :scram256, :mongodb_cr, :mongodb_x509, :gssapi, :aws, :plain. + # MongoDB Server defaults to :scram, which will use "SCRAM-SHA-256" if available, + # otherwise fallback to "SCRAM-SHA-1" (:scram256 will always use "SCRAM-SHA-256".) + # This setting is handled by the MongoDB Ruby Driver. Please refer to: + # https://mongodb.com/docs/ruby-driver/current/reference/authentication/ + # auth_mech: :scram + + # The database or source to authenticate the user against. + # (default: the database specified above or admin) + # auth_source: admin + + # Force the driver cluster to behave in a certain manner instead of auto-discovering. + # Can be one of: :direct, :replica_set, :sharded. Set to :direct + # when connecting to hidden members of a replica set. + # connect: :direct + + # Changes the default time in seconds the server monitors refresh their status + # via hello commands. (default: 10) + # heartbeat_frequency: 10 + + # The time in seconds for selecting servers for a near read preference. (default: 0.015) + # local_threshold: 0.015 + + # The timeout in seconds for selecting a server for an operation. (default: 30) + # server_selection_timeout: 30 + + # The maximum number of connections in the connection pool. (default: 5) + # max_pool_size: 5 + + # The minimum number of connections in the connection pool. (default: 1) + # min_pool_size: 1 + + # The time to wait, in seconds, in the connection pool for a connection + # to be checked in before timing out. (default: 5) + # wait_queue_timeout: 5 + + # The time to wait to establish a connection before timing out, in seconds. + # (default: 10) + # connect_timeout: 10 + + # How long to wait for a response for each operation sent to the + # server. This timeout should be set to a value larger than the + # processing time for the longest operation that will be executed + # by the application. Note that this is a client-side timeout; + # the server may continue executing an operation after the client + # aborts it with the SocketTimeout exception. + # (default: nil, meaning no timeout) + # socket_timeout: 5 + + # The name of the replica set to connect to. Servers provided as seeds that do + # not belong to this replica set will be ignored. + # replica_set: name + + # Compressors to use for wire protocol compression. (default is to not use compression) + # "zstd" requires zstd-ruby gem. "snappy" requires snappy gem. + # Refer to: https://www.mongodb.com/docs/ruby-driver/current/reference/create-client/#compression + # compressors: ["zstd", "snappy", "zlib"] + + # Whether to connect to the servers via ssl. (default: false) + # ssl: true + + # The certificate file used to identify the connection against MongoDB. + # ssl_cert: /path/to/my.cert + + # The private keyfile used to identify the connection against MongoDB. + # Note that even if the key is stored in the same file as the certificate, + # both need to be explicitly specified. + # ssl_key: /path/to/my.key + + # A passphrase for the private key. + # ssl_key_pass_phrase: password + + # Whether to do peer certification validation. (default: true) + # ssl_verify: true + + # The file containing concatenated certificate authority certificates + # used to validate certs passed from the other end of the connection. + # ssl_ca_cert: /path/to/ca.cert + + # Whether to truncate long log lines. (default: true) + # truncate_logs: true + + # Configure Mongoid-specific options. (optional) + options: + # Allow BSON::Decimal128 to be parsed and returned directly in + # field values. When BSON 5 is present and the this option is set to false + # (the default), BSON::Decimal128 values in the database will be returned + # as BigDecimal. + # + # @note this option only has effect when BSON 5+ is present. Otherwise, + # the setting is ignored. + # allow_bson5_decimal128: false + + # When this flag is false, named scopes cannot unset a default scope. + # This is the traditional (and default) behavior in Mongoid 9 and earlier. + # + # Setting this flag to true will allow named scopes to unset the default + # scope. This will be the default in Mongoid 10. + # + # See https://jira.mongodb.org/browse/MONGOID-5785 for more details. + # allow_scopes_to_unset_default_scope: false + + # Application name that is printed to the MongoDB logs upon establishing + # a connection. Note that the name cannot exceed 128 bytes in length. + # It is also used as the database name if the database name is not + # explicitly defined. + # app_name: nil + + # When this flag is false, callbacks for embedded documents will not be + # called. This is the default in 9.0. + # + # Setting this flag to true restores the pre-9.0 behavior, where callbacks + # for embedded documents are called. This may lead to stack overflow errors + # if there are more than cicrca 1000 embedded documents in the root + # document's dependencies graph. + # See https://jira.mongodb.org/browse/MONGOID-5658 for more details. + # around_callbacks_for_embeds: false + + # Sets the async_query_executor for the application. By default the thread pool executor + # is set to `:immediate. Options are: + # + # - :immediate - Initializes a single +Concurrent::ImmediateExecutor+ + # - :global_thread_pool - Initializes a single +Concurrent::ThreadPoolExecutor+ + # that uses the +async_query_concurrency+ for the +max_threads+ value. + # async_query_executor: :immediate + + # Mark belongs_to associations as required by default, so that saving a + # model with a missing belongs_to association will trigger a validation + # error. + # belongs_to_required_by_default: true + + # Set the global discriminator key. + # discriminator_key: "_type" + + # Raise an exception when a field is redefined. + # duplicate_fields_exception: false + + # Defines how many asynchronous queries can be executed concurrently. + # This option should be set only if `async_query_executor` is set + # to `:global_thread_pool`. + # global_executor_concurrency: nil + + # When this flag is true, any attempt to change the _id of a persisted + # document will raise an exception (`Errors::ImmutableAttribute`). + # This is the default in 9.0. Setting this flag to false restores the + # pre-9.0 behavior, where changing the _id of a persisted + # document might be ignored, or it might work, depending on the situation. + # immutable_ids: true + + # Include the root model name in json serialization. + # include_root_in_json: false + + # # Include the _type field in serialization. + # include_type_for_serialization: false + + # Whether to join nested persistence contexts for atomic operations + # to parent contexts by default. + # join_contexts: false + + # When this flag is false (the default as of Mongoid 9.0), a document that + # is created or loaded will remember the storage options that were active + # when it was loaded, and will use those same options by default when + # saving or reloading itself. + # + # When this flag is true you'll get pre-9.0 behavior, where a document will + # not remember the storage options from when it was loaded/created, and + # subsequent updates will need to explicitly set up those options each time. + # + # For example: + # + # record = Model.with(collection: 'other_collection') { Model.first } + # + # This will try to load the first document from 'other_collection' and + # instantiate it as a Model instance. Pre-9.0, the record object would + # not remember that it came from 'other_collection', and attempts to + # update it or reload it would fail unless you first remembered to + # explicitly specify the collection every time. + # + # As of Mongoid 9.0, the record will remember that it came from + # 'other_collection', and updates and reloads will automatically default + # to that collection, for that record object. + # legacy_persistence_context_behavior: false + + # When this flag is false, a document will become read-only only once the + # #readonly! method is called, and an error will be raised on attempting + # to save or update such documents, instead of just on delete. When this + # flag is true, a document is only read-only if it has been projected + # using #only or #without, and read-only documents will not be + # deletable/destroyable, but they will be savable/updatable. + # When this feature flag is turned on, the read-only state will be reset on + # reload, but when it is turned off, it won't be. + # legacy_readonly: false + + # The log level. + # + # It must be set prior to referencing clients or Mongo.logger, + # changes to this option are not be propagated to any clients and + # loggers that already exist. + # + # Additionally, only when the clients are configured via the + # configuration file is the log level given by this option honored. + # log_level: :info + + # Store BigDecimals as Decimal128s instead of strings in the db. + # map_big_decimal_to_decimal128: true + + # Preload all models in development, needed when models use inheritance. + # preload_models: false + + # When this flag is true, callbacks for every embedded document will be + # called only once, even if the embedded document is embedded in multiple + # documents in the root document's dependencies graph. + # This is the default in 9.0. Setting this flag to false restores the + # pre-9.0 behavior, where callbacks are called for every occurrence of an + # embedded document. The pre-9.0 behavior leads to a problem that for multi + # level nested documents callbacks are called multiple times. + # See https://jira.mongodb.org/browse/MONGOID-5542 + # prevent_multiple_calls_of_embedded_callbacks: true + + # Raise an error when performing a #find and the document is not found. + # raise_not_found_error: true + + # Raise an error when defining a scope with the same name as an + # existing method. + # scope_overwrite_exception: false + + # Return stored times as UTC. + # use_utc: false + + # Configure Driver-specific options. (optional) + driver_options: + # When this flag is off, an aggregation done on a view will be executed over + # the documents included in that view, instead of all documents in the + # collection. When this flag is on, the view fiter is ignored. + # broken_view_aggregate: true + + # When this flag is set to false, the view options will be correctly + # propagated to readable methods. + # broken_view_options: true + + # When this flag is set to true, the update and replace methods will + # validate the paramters and raise an error if they are invalid. + # validate_update_replace: false + +test: + clients: + default: + database: library_test + hosts: + - localhost:27017 + options: + read: + mode: :primary + max_pool_size: 1 diff --git a/config/routes.rb b/config/routes.rb index ae608d2..8158d53 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -27,4 +27,6 @@ # Render dynamic PWA files from app/views/pwa/* get 'service-worker' => 'rails/pwa#service_worker', as: :pwa_service_worker get 'manifest' => 'rails/pwa#manifest', as: :pwa_manifest + + get 'books(/:page)', to: 'books#index' end diff --git a/docker-compose.yml b/docker-compose.yml index 9c195af..b7bf75d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,5 +33,18 @@ services: POSTGRES_HOST: 'db' POSTGRES_USER: 'postgres' POSTGRES_PASSWORD: 'postgres' + + mongodb: + image: mongo:latest + ports: + - '27017:27017' + volumes: + - mongodb:/data/db + healthcheck: + test: ["CMD","mongosh", "--eval", "db.adminCommand('ping')"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 5s volumes: db: diff --git a/lib/tasks/mongo_inpx.rake b/lib/tasks/mongo_inpx.rake new file mode 100644 index 0000000..da27c4b --- /dev/null +++ b/lib/tasks/mongo_inpx.rake @@ -0,0 +1,121 @@ +namespace :mongo_inpx do + desc 'Извлекаем содержимое inpx-файла' + task :ls, [ :path ] => [ :environment ] do |task, args| + args.with_defaults(path: ENV.fetch('INPX_PATH', 'db/data/archive.inpx')) + Zip::File.open(args[:path]) do |zip_file| + zip_file.each do |entry| + puts "#{entry.name}: #{entry.size}" + end + end + end + + desc 'Перестриваем базу данных по inpx-индексу' + task :rebuild, [ :path ] => [ :environment ] do |task, args| + start = Time.zone.now + args.with_defaults(path: ENV.fetch('INPX_PATH', 'db/data/archive.inpx')) + + FileUtils.rm_rf('tmp/extracts/.', secure: true) + FileUtils.mkdir_p('tmp/extracts') + + Zip::File.open(args[:path]) do |zip_file| + zip_file.each do |entry| + entry.extract("tmp/extracts/#{entry}") + end + end + + Mongo::Book.destroy_all + Seeds::Mongo::GenreLoad.call(filename: 'db/seeds/genres.yml') + puts "Жанры #{Genre.count}" + Seeds::Mongo::LanguageLoad.call(filename: 'db/seeds/languages.yml') + puts "Языки #{Language.count}" + + regexp_files = /\Atmp\/extracts\/(usr|fb2)-(ru|en)/ + files = Dir.glob('tmp/extracts/*.inp').select { |entry| entry =~ regexp_files } + + files_count = Dir.glob('tmp/extracts/*.inp').count + puts "Общее количество inp-файлов #{files_count}" + puts "Отобранных для развертывания inp-файлов #{files.count}" + + lines = Seeds::LinesFromInpx.call(files: files) + count = Settings.app.index_concurrent_processes + chunks = lines.in_groups(count, false) + pp chunks.count + + title = 'Папки' + puts title + folders = Parallel.map(Range.new(0, count - 1), in_processes: count) do |index| + start = Time.zone.now + folders = Folders::Parse.call(books: chunks[index]) + finish = Time.zone.now + puts format("Процесс %d: %0.2f сек", index, (finish - start)) + folders.uniq + end + Mongo::Folder.collection.insert_many(folders.flatten.uniq.map { |f| { name: f } }) + puts "#{title} #{Folder.count}" + + title = 'Книги' + puts title + Parallel.map(Range.new(0, count - 1), in_processes: count) do |index| + start = Time.zone.now + Mongo::Books::Parse.call(books: chunks[index]) + finish = Time.zone.now + puts format("Процесс %d: %0.2f сек", index, (finish - start)) + end + puts "#{title} #{Mongo::Book.count}" + + title = 'Авторы' + puts title + authors = Parallel.map(Range.new(0, count - 1), in_processes: count) do |index| + start = Time.zone.now + authors = Authors::Parse.call(books: chunks[index]) + finish = Time.zone.now + puts format("Процесс %d: %0.2f сек", index, (finish - start)) + authors.flatten.uniq + end + Author.import authors.flatten.uniq, validate: false + puts "#{title} #{Author.count}" + + title = 'Ключевые слова' + puts title + keywords = Parallel.map(Range.new(0, count - 1), in_processes: count) do |index| + start = Time.zone.now + keywords = Keywords::Parse.call(books: chunks[index]) + finish = Time.zone.now + puts format("Процесс %d: %0.2f сек", index, (finish - start)) + keywords.uniq + end + Keyword.import keywords.flatten.uniq.map { |f| { name: f } }, validate: false + puts "#{title} #{Keyword.count}" + + puts 'Связи' + + # TODO видимо тут не хватает сервис-объекта + genres_map = Genre.pluck(:slug, :id).to_h + keywords_map = Keyword.pluck(:name, :id).to_h + authors_map = Author.pluck(:original, :id).to_h + + Parallel.map(Range.new(0, count - 1), in_processes: count) do |index| + start = Time.zone.now + Books::Links.call( + books: chunks[index], + genres_map: genres_map, + keywords_map: keywords_map, + authors_map: authors_map + ) + finish = Time.zone.now + puts format("Процесс %d: %0.2f сек", index, (finish - start)) + end + + Authors::CounterCache.call + Genres::CounterCache.call + Keywords::CounterCache.call + Languages::CounterCache.call + + puts "Связей книг и авторов #{BooksAuthor.count}" + puts "Связей книг и ключевых слов #{BooksKeyword.count}" + puts "Связей книг и жанров #{BooksGenre.count}" + + FileUtils.rm_rf('tmp/extracts/.', secure: true) + puts "Время создания индекса #{Time.zone.now - start}" + end +end