====== Base services ======
===== Nginx and PHP =====
Install Nginx from package
apt install nginx
systemctl status nginx
nginx -v
nginx version: nginx/1.18.0 (Ubuntu)
Install PHP-FPM
apt install php-fpm php-mysql php-gd php-apcu php-gmp php-curl php-intl php-mbstring php-xmlrpc php-xml php-cli php-zip
cd /etc/php/7.4/fpm/
cp php.ini php.ini.ORI
nano -w php.ini
date.timezone = Europe/Rome
memory_limit = 256M
post_max_size = 512M
upload_max_filesize = 512M
max_execution_time = 600
cgi.fix_pathinfo=0
systemctl restart php7.4-fpm
systemctl status php7.4-fpm
php --version
PHP 7.4.3 (cli) (built: Oct 6 2020 15:47:56) ( NTS )
Copyright (c) The PHP Group
Zend Engine v3.4.0, Copyright (c) Zend Technologies
with Zend OPcache v7.4.3, Copyright (c), by Zend Technologies
Add Nginx virtual host for Drupal9 (also see [[https://github.com/esmero/archipelago-deployment/blob/1.0.0-RC2D9/nginxconfigford8/default.conf|Archipelago deployment]])
mkdir /var/www/html/web
cd /etc/nginx/sites-available/
nano -w drupal9
server {
server_name dbopen.ba.cnr.it;
client_max_body_size 512M; ## Match with PHP from FPM container
root /var/www/html/web; ## <-- Your only path reference.
fastcgi_send_timeout 120s;
fastcgi_read_timeout 120s;
fastcgi_pass_request_headers on;
fastcgi_buffers 16 16k;
fastcgi_buffer_size 32k;
location = /favicon.ico {
log_not_found off;
access_log off;
}
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
# Very rarely should these ever be accessed outside of your lan
location ~* \.(txt|log)$ {
deny all;
}
location ~ \..*/.*\.php$ {
return 403;
}
location ~ ^/sites/.*/private/ {
return 403;
}
# Allow "Well-Known URIs" as per RFC 5785
location ~* ^/.well-known/ {
allow all;
}
# Block access to "hidden" files and directories whose names begin with a
# period. This includes directories used by version control systems such
# as Subversion or Git to store control files.
location ~ (^|/)\. {
return 403;
}
location / {
try_files $uri /index.php?$query_string; # For Drupal >= 7
}
location @rewrite {
rewrite ^/(.*)$ /index.php?q=$1;
}
# Don't allow direct access to PHP files in the vendor directory.
location ~ /vendor/.*\.php$ {
deny all;
return 404;
}
# In Drupal 8, we must also match new paths where the '.php' appears in
# the middle, such as update.php/selection. The rule we use is strict,
# and only allows this pattern with the update.php front controller.
# This allows legacy path aliases in the form of
# blog/index.php/legacy-path to continue to route to Drupal nodes. If
# you do not have any paths like that, then you might prefer to use a
# laxer rule, such as:
# location ~ \.php(/|$) {
# The laxer rule will continue to work if Drupal uses this new URL
# pattern with front controllers other than update.php in a future
# release.
location ~ '\.php$|^/update.php' {
fastcgi_split_path_info ^(.+?\.php)(|/.*)$;
include fastcgi_params;
# Block httpoxy attacks. See https://httpoxy.org/.
fastcgi_param HTTP_PROXY "";
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_param PHP_VALUE "upload_max_filesize=512M \n post_max_size=512M";
proxy_read_timeout 900s;
fastcgi_intercept_errors on;
fastcgi_pass unix:/var/run/php/php7.4-fpm.sock;
}
# Fighting with Styles? This little gem is amazing.
location ~ ^/sites/.*/files/styles/ { # For Drupal >= 7
try_files $uri @rewrite;
}
# Handle private files through Drupal.
location ~ ^/system/files/ { # For Drupal >= 7
try_files $uri /index.php?$query_string;
}
}
ln -s /etc/nginx/sites-available/drupal9 /etc/nginx/sites-enabled/
nginx -t
systemctl restart nginx
Add certificate and force HTTPS.
apt install certbot python3-certbot-nginx
certbot --nginx
cd /etc/nginx/sites-enabled/
rm default
Enable http2 on Nginx.
nano -w /etc/nginx/sites-available/drupal9
listen 443 ssl http2;
systemctl restart nginx
===== MariaDB database =====
Install MariaDB database server.
apt install mariadb-server mariadb-client
systemctl status mysql
● mariadb.service - MariaDB 10.3.25 database server
mysql_secure_installation
Set a root password? [Y/n] Y
Remove anonymous users? [Y/n] Y
Disallow root login remotely? [Y/n] Y
Remove test database and access to it? [Y/n] Y
Reload privilege tables now? [Y/n] Y
Create database and user for Drupal.
sudo mysql -u root -p
CREATE DATABASE drupal_db;
CREATE USER 'drupal_user'@'localhost' IDENTIFIED BY 'some_strong_password';
GRANT ALL PRIVILEGES ON drupal_db.* TO 'drupal_user'@'localhost' WITH GRANT OPTION;
flush privileges;
quit;
===== Composer =====
Install composer globally (see [[https://getcomposer.org/download/|Composer official site]])
php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
php -r "if (hash_file('sha384', 'composer-setup.php') === '756890a4488ce9024fc62c56153228907f1545c228516cbf63f885e036d37e9a59d27d63f46af1d4d07ee0f76181c7d3') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
php composer-setup.php --filename=composer --install-dir=/usr/local/bin
All settings correct for using Composer
Downloading...
Composer (version 2.0.11) successfully installed to: /usr/local/bin/composer
Use it: php /usr/local/bin/composer
rm composer-setup.php
sudo -u www-data composer --version
Composer version 2.0.11 2021-02-24 14:57:23
===== Minio =====
Download and install
wget https://dl.minio.io/server/minio/release/linux-amd64/minio
chmod +x minio
sudo mv minio /usr/local/bin/
$ minio --version
minio version RELEASE.2021-03-01T04-20-55Z
Prepare user/group for daemon
sudo groupadd --system minio
sudo useradd -s /sbin/nologin --system -g minio minio
sudo usermod -L minio
sudo chage -E0 minio
Create Minio default settings
sudo nano -w /etc/default/minio
# Volume to be used for Minio server.
MINIO_VOLUMES="/srv/repo"
# Use if you want to run Minio on a custom port.
MINIO_OPTS="-C /etc/minio --address localhost:9000"
# Access Key of the server.
MINIO_ACCESS_KEY=minioaccess
# Secret key of the server.
MINIO_SECRET_KEY=miniosecret
Set folder and permissions
sudo chown -R minio:minio /srv/repo
sudo mkdir /etc/minio
sudo chown minio:minio /etc/minio
Set Minio service
sudo nano -w /etc/systemd/system/minio.service
[Unit]
Description=Minio
Documentation=https://docs.minio.io
Wants=network-online.target
After=network-online.target
AssertFileIsExecutable=/usr/local/bin/minio
[Service]
WorkingDirectory=/srv/repo
User=minio
Group=minio
EnvironmentFile=-/etc/default/minio
ExecStartPre=/bin/bash -c "if [ -z \"${MINIO_VOLUMES}\" ]; then echo \"Variable MINIO_VOLUMES not set in /etc/default/minio\"; exit 1; fi"
ExecStart=/usr/local/bin/minio server $MINIO_OPTS $MINIO_VOLUMES
# Let systemd restart this service always
Restart=always
# Specifies the maximum file descriptor number that can be opened by this process
LimitNOFILE=65536
# Specifies the maximum number of threads this process can create
TasksMax=infinity
# Disable timeout logic and wait until process is stopped
TimeoutStopSec=infinity
SendSIGKILL=no
[Install]
WantedBy=multi-user.target
Enable and run service
sudo systemctl daemon-reload
sudo systemctl enable minio
sudo systemctl start minio
sudo systemctl status minio
Add reverse proxy for Minio to Nginx.
sudo nano -w /etc/nginx/sites-enabled/drupal9
# Proxy requests to MinIO server running on port 9000
location /minio/ {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://localhost:9000;
}
Check browsing https://dbopen.ba.cnr.it/minio
===== Cantaloupe =====
We need Java.
apt install openjdk-14-jdk
java -version
openjdk version "14.0.2" 2020-07-14
OpenJDK Runtime Environment (build 14.0.2+12-Ubuntu-120.04)
OpenJDK 64-Bit Server VM (build 14.0.2+12-Ubuntu-120.04, mixed mode, sharing)
Compile Cantaloupe 5.0-SNAPSHOT from develop branch.
apt install maven (needed only if you have to compile cantaloupe)
$ wget https://github.com/cantaloupe-project/cantaloupe/archive/develop.zip
$ unzip develop.zip
$ cd cantaloupe-develop
$ mvn clean package -DskipTests
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 07:47 min
[INFO] Finished at: 2021-03-04T13:10:32Z
[INFO] ------------------------------------------------------------------------
Install Cantaloupe
$ mv cantaloupe-develop/target/cantaloupe-5.0-SNAPSHOT.zip ./
$ unzip cantaloupe-5.0-SNAPSHOT.zip
$ sudo mv cantaloupe-5.0-SNAPSHOT /opt/
$ sudo mkdir /srv/canta/cache /srv/canta/log /srv/canta/home /srv/canta/tmp
$ sudo useradd -d /srv/canta/home -s /bin/false cantaloupe
$ sudo chown -R cantaloupe:cantaloupe /opt/cantaloupe-5.0-SNAPSHOT /srv/canta
$ sudo cp /opt/cantaloupe-5.0-SNAPSHOT/deps/Linux-x86-64/lib/libkdu_* /usr/lib/
Configure
cd /opt/cantaloupe-5.0-SNAPSHOT
sudo cp delegates.rb.sample delegates.rb (needed for a probably bug?)
sudo cp cantaloupe.properties.sample cantaloupe.properties
sudo nano -w cantaloupe.properties
Settings difference from .sample
temp_pathname = /srv/canta/tmp
max_pixels = 400000000
delegate_script.enabled = true
endpoint.admin.enabled = true
endpoint.admin.secret = admin_secret
source.static = S3Source
FilesystemSource.BasicLookupStrategy.path_prefix = /srv/canta/tmp/
S3Source.endpoint =http://127.0.0.1:9000
S3Source.access_key_id =access_key
S3Source.secret_key =secret_key
S3Source.BasicLookupStrategy.bucket.name =archipelago
S3Source.BasicLookupStrategy.path_prefix =/
processor.selection_strategy = ManualSelectionStrategy
processor.fallback_retrieval_strategy = CacheStrategy
processor.pdf.scratch_file_enabled = true
processor.pdf.max_memory_bytes = 500MB
cache.server.derivative.enabled = true
cache.server.derivative =FilesystemCache
cache.server.purge_missing = true
cache.server.worker.enabled = true
FilesystemCache.pathname = /srv/canta/cache
HeapCache.persist.filesystem.pathname = /srv/canta/heap.cache
log.application.level = trace
log.application.ConsoleAppender.enabled = false
log.application.FileAppender.pathname = /srv/canta/log/application.log
log.application.RollingFileAppender.enabled = true
log.application.RollingFileAppender.pathname = /srv/canta/log/application.log
log.application.RollingFileAppender.TimeBasedRollingPolicy.filename_pattern = /srv/canta/log/application-%d{yyyy-MM-dd}.lo
log.error.FileAppender.pathname = /srv/canta/log/error.log
log.error.RollingFileAppender.enabled = true
log.error.RollingFileAppender.pathname = /srv/canta/log/error.log
log.error.RollingFileAppender.TimeBasedRollingPolicy.filename_pattern = /srv/canta/log/error-%d{yyyy-MM-dd}.log
log.access.FileAppender.pathname = /srv/canta/log/access.log
log.access.RollingFileAppender.pathname = /srv/canta/log/access.log
log.access.RollingFileAppender.TimeBasedRollingPolicy.filename_pattern = /srv/canta/log/access-%d{yyyy-MM-dd}.log
Set Cantaloupe as a service
sudo nano -w /etc/systemd/system/cantaloupe.service
[Unit]
Description=Cantaloupe Image Server 5.0 SNAPSHOT
[Service]
Type=simple
User=cantaloupe
ExecStart=/usr/bin/java -Dcantaloupe.config=/opt/cantaloupe-5.0-SNAPSHOT/cantaloupe.properties -Xmx6g -jar /opt/cantaloupe-5.0-SNAPSHOT/cantaloupe-5.0-SNAPSHOT.jar
ExecStop=/usr/bin/killall -9 cantaloupe-5.0-SNAPSHOT.jar
Restart=on-failure
[Install]
WantedBy=multi-user.target
sudo systemctl daemon-reload
sudo systemctl enable cantaloupe
Enable Nginx as reverse proxy for Cantaloupe
# Proxy requests to the Cantaloupe server running on port 8182
location /cantaloupe/ {
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Forwarded-Path /cantaloupe/;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
if ($request_uri ~* "/cantaloupe/(.*)") {
proxy_pass http://127.0.0.1:8182/$1;
}
}
Start Cantaloupe service
sudo systemctl start cantaloupe
Browse to check here https://dbopen.ba.cnr.it/cantaloupe/
\\
\\
[[#cantaupdate5|Update Cantaloupe to 5.0 as this version was released.]]
wget https://github.com/cantaloupe-project/cantaloupe/releases/download/v5.0/cantaloupe-5.0.zip
unzip cantaloupe-5.0.zip
sudo mv cantaloupe-5.0 /opt/
cd /opt/cantaloupe-5.0
cp ../cantaloupe-5.0-SNAPSHOT/cantaloupe.properties ./
cp delegates.rb.sample delegates.rb
sudo cp /etc/systemd/system/cantaloupe.service ./
sudo nano -w cantaloupe.service
[Unit]
Description=Cantaloupe Image Server 5.0
[Service]
Type=simple
User=cantaloupe
ExecStart=/usr/bin/java -Dcantaloupe.config=/opt/cantaloupe-5.0/cantaloupe.properties -Xmx6g -jar /opt/cantaloupe-5.0/cantaloupe-5.0.jar
ExecStop=/usr/bin/killall -9 cantaloupe-5.0.jar
Restart=on-failure
[Install]
WantedBy=multi-user.target
sudo chown -R cantaloupe:cantaloupe /opt/cantaloupe-5.0
sudo systemctl stop cantaloupe
sudo cp cantaloupe.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable cantaloupe
sudo systemctl start cantaloupe
===== SOLR =====
Edit system settings for Solr
# sysctl vm.max_map_count
vm.max_map_count = 65530
# nano -w /etc/sysctl.d/10-solr.conf
# map_count should be around 1 per 128 KB of system memory
# we have 16GB system memory so
vm.max_map_count=131060
# service procps restart
# sysctl vm.max_map_count
vm.max_map_count = 131060
$ ulimit -n
1024
$ ulimit -u
63709
$ sudo nano -w /etc/systemd/system.conf
DefaultLimitNOFILE=65535:524288
DefaultLimitNPROC=65535
$ sudo nano -w /etc/systemd/user.conf
DefaultLimitNOFILE=65535:524288
DefaultLimitNPROC=65535
$ sudo nano -w /etc/security/limits.conf
* hard nofile 65535
* soft nofile 65535
* hard nproc 65535
* soft nproc 65535
reboot
$ ulimit -n
65535
$ ulimit -u
65535
Install Solr standalone
wget https://downloads.apache.org/lucene/solr/8.8.1/solr-8.8.1.tgz
tar xzf solr-8.8.1.tgz solr-8.8.1/bin/install_solr_service.sh --strip-components=2
sudo bash ./install_solr_service.sh solr-8.8.1.tgz -i /opt -d /srv/solr -u solr -s solr -p 8983
id: ‘solr’: no such user
Creating new user: solr
Warning: The home dir /srv/solr you specified already exists.
Adding system user `solr' (UID 113) ...
Adding new group `solr' (GID 119) ...
Adding new user `solr' (UID 113) with group `solr' ...
The home directory `/srv/solr' already exists. Not copying from `/etc/skel'.
adduser: Warning: The home directory `/srv/solr' does not belong to the user you are currently creating.
Extracting solr-8.8.1.tgz to /opt
Installing symlink /opt/solr -> /opt/solr-8.8.1 ...
Installing /etc/init.d/solr script ...
Installing /etc/default/solr.in.sh ...
Service solr installed.
Customize Solr startup configuration in /etc/default/solr.in.sh
● solr.service - LSB: Controls Apache Solr as a Service
Loaded: loaded (/etc/init.d/solr; generated)
Active: active (exited) since Fri 2021-03-05 10:06:51 UTC; 5s ago
Docs: man:systemd-sysv-generator(8)
Process: 2780 ExecStart=/etc/init.d/solr start (code=exited, status=0/SUCCESS)
Mar 05 10:06:43 dbopen systemd[1]: Starting LSB: Controls Apache Solr as a Service...
Mar 05 10:06:44 dbopen su[2784]: (to solr) root on none
Mar 05 10:06:44 dbopen su[2784]: pam_unix(su-l:session): session opened for user solr by (uid=0)
Mar 05 10:06:50 dbopen solr[2864]: Started Solr server on port 8983 (pid=2856). Happy searching!
Mar 05 10:06:51 dbopen systemd[1]: Started LSB: Controls Apache Solr as a Service.
Some basic settings (Memory and IP allow/deny
$ sudo systemctl stop solr
$ sudo nano -w /etc/default/solr.in.sh
SOLR_PID_DIR="/srv/solr"
SOLR_HOME="/srv/solr/data"
LOG4J_PROPS="/srv/solr/log4j2.xml"
SOLR_LOGS_DIR="/srv/solr/logs"
SOLR_PORT="8983"
SOLR_HEAP="4g"
SOLR_IP_WHITELIST="127.0.0.1"
Compile solr-ocrhighlighting from HEAD branch
An issue with plugin compiled from HEAD, so revert to 0.5.0 make all works well.
Issue solved so no more needs to revert.
$ wget https://github.com/dbmdz/solr-ocrhighlighting/archive/main.zip
$ unzip main.zip
$ cd solr-ocrhighlighting-main/
$ JAVA_HOME=/usr/lib/jvm/java-14-openjdk-amd64 mvn package
[INFO] Building jar: /home/user/solr-ocrhighlighting-main/target/solr-ocrhighlighting-0.6.0-SNAPSHOT-javadoc.jar
[INFO]
[INFO] --- maven-source-plugin:3.2.1:jar-no-fork (attach-sources) @ solr-ocrhighlighting ---
[INFO] Building jar: /home/user/solr-ocrhighlighting-main/target/solr-ocrhighlighting-0.6.0-SNAPSHOT-sources.jar
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 45.214 s
[INFO] Finished at: 2021-03-05T11:15:18Z
[INFO] ------------------------------------------------------------------------
sudo mkdir /opt/solr/contrib/archipelago
sudo mkdir /opt/solr/contrib/archipelago/lib
sudo mv target/solr-ocrhighlighting-0.6.0-SNAPSHOT.jar /opt/solr/contrib/archipelago/lib/
We need last ocrhighlighting from master (0.7.0 SNAPSHOT)
$ wget https://github.com/dbmdz/solr-ocrhighlighting/archive/main.zip
$ unzip main.zip
$ cd solr-ocrhighlighting-main/
$ JAVA_HOME=/usr/lib/jvm/java-14-openjdk-amd64 mvn package
$ cd ..
$ sudo mv /opt/solr/contrib/archipelago/lib/solr-ocrhighlighting-0.6.0-SNAPSHOT.jar ./
$ sudo mv solr-ocrhighlighting-main/target/solr-ocrhighlighting-0.7.0-SNAPSHOT.jar /opt/solr/contrib/archipelago/lib/
$ rm -R solr-ocrhighlighting-main
Reload Solr
Update to ocrhighlighting 0.7.0 release
$ wget https://github.com/dbmdz/solr-ocrhighlighting/archive/refs/tags/0.7.0.zip
$ unzip 0.7.0.zip
$ cd solr-ocrhighlighting-0.7.0
$ JAVA_HOME=/usr/lib/jvm/java-14-openjdk-amd64 mvn package
$ cd ..
$ sudo mv /opt/solr/contrib/archipelago/lib/solr-ocrhighlighting-0.7.0-SNAPSHOT.jar ./
$ sudo mv solr-ocrhighlighting-0.7.0/target/solr-ocrhighlighting-0.7.0.jar /opt/solr/contrib/archipelago/lib/
$ rm -R solr-ocrhighlighting-0.7.0
Reload Solr
Create archipelago core
$ cd /opt/solr
$ sudo -u solr bin/solr create_core -c archipelago -V
Copying configuration to new core instance directory:
/srv/solr/data/archipelago
Creating new core 'archipelago' using command:
http://localhost:8983/solr/admin/cores?action=CREATE&name=archipelago&instanceDir=archipelago
{
"responseHeader":{
"status":0,
"QTime":2174},
"core":"archipelago"}
Replace core conf with Archipelago Solr conf
$ sudo systemctl stop solr
$ wget https://github.com/esmero/archipelago-deployment/archive/1.0.0-RC2D9.zip
$ unzip 1.0.0-RC2D9.zip
$ sudo -u solr mv /srv/solr/data/archipelago/conf/solrconfig.xml /srv/solr/data/archipelago/conf/solrconfig.xml.ORI
$ sudo -u solr cp archipelago-deployment-1.0.0-RC2D9/persistent/solrconfig/conf/* /srv/solr/data/archipelago/conf/
$ sudo systemctl start solr
[[#solrita|Edit Solr filters for Italian text language]]
$ sudo -u solr nano -w /srv/solr/data/archipelago/conf/schema_extra_types.xml
[[#solrsugg|Edit Solr schema for Suggester split on word without punctuation]]
$ sudo -u solr nano -w /srv/solr/data/archipelago/conf/schema.xml