Database and Middleware Deployment
8/29/25About 4 minInstallation and Deployment
1. DM8 Database
1.1 Create the mount directory
# Create the mount directory
mkdir /data/dm8
# Grant permissions to the mount directory
chmod -R 777 /data/dm81.1 Write the docker-compose.yml script file
sudo vi docker-compose.ymlFill in the following content and save the file.
version: "3.2"
services:
dm8:
privileged: true
hostname: dm8
image: dm8:dm8_20250506_x86_rh7_64 # Contact the Dameng vendor or build a test image yourself; non-Docker deployment is also supported
restart: always
ports:
- "5236:5236"
volumes:
- /data/dm8:/home/dmdba/data # Mount the data storage directory
environment:
- TZ=Asia/Shanghai
- CASE_SENSITIVE=0 # Set database case-insensitive mode
- SYSDBA_PWD=InC3tmU4bijT4vkl # Password for the SYSDBA user
- SYSAUDITOR_PWD=InC3tmU4bijT4vkl # Password for the SYSAUDITOR user1.2 Start
docker-compose up -d2. PostgreSQL Database
2.1 Create the mount directory
# Create the mount directory
mkdir /data/postgresql
# Grant permissions to the mount directory
chmod -R 777 /data/postgresql2.3 Write the docker-compose.yml script file
sudo vi docker-compose.ymlFill in the following content and save the file.
version: "3.8"
services:
dolphinscheduler-postgresql:
privileged: true
restart: always
image: bitnami/postgresql:15.2.0
container_name: dolphinscheduler-postgresql
ports:
- "5432:5432"
environment:
POSTGRESQL_USERNAME: root # Database username
POSTGRESQL_PASSWORD: Idgf~V4pco&PF # Database password
POSTGRESQL_DATABASE: dolphinscheduler # Database name
volumes:
- /data/postgresql:/bitnami/postgresql # Mount the data storage directory2.3 Start
docker-compose up -d3. Redis
3.1 Write the docker-compose.yml script file
sudo vi docker-compose.ymlFill in the following content and save the file.
version: "3.2"
services:
redis:
privileged: true
restart: always
image: redis
container_name: redis
command: redis-server --requirepass J98%FHF#9h@e88h9fre9 # Redis password
ports:
- "6379:6379"3.2 Start
docker-compose up -d4. RabbitMQ
4.1 Write the docker-compose.yml script file
sudo vi docker-compose.ymlFill in the following content and save the file.
version: '3.3'
services:
rabbitmq:
restart: always
privileged: true
image: rabbitmq:3.12-management
ports:
- 5672:5672
- 15672:15672
environment:
- RABBITMQ_DEFAULT_USER=admin
- RABBITMQ_DEFAULT_PASS=Ej^iUNFLp9MQouc14.2 Start
docker-compose up -d5. Nginx
5.1 Create the www directory
sudo mkdir www5.2 Create nginx.conf
sudo vi nginx.confFill in the following content and save the file.
user root;
worker_processes 8;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 3000;
}
http {
include /etc/nginx/mime.types;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
server_tokens off;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 300;
keepalive_requests 1000;
client_header_timeout 60;
client_body_timeout 60;
proxy_buffering on;
proxy_buffer_size 4k;
proxy_buffers 8 16k;
proxy_busy_buffers_size 32k;
proxy_max_temp_file_size 0;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_comp_level 1;
gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;
gzip_vary off;
gzip_disable "MSIE [1-6]\.";
include http_vhost/*.conf;
}5.3 Create the http_vhost directory
sudo mkdir http_vhost5.4 Create qdata.conf
sudo vi qdata.confFill in the following content and save the file. Adjust the IP address in the content below to the server's private IP, for example 192.168.3.35.
server {
listen 8082;
server_name 192.168.3.35;
client_max_body_size 500M;
add_header Strict-Transport-Security "max-age=31536000";
proxy_connect_timeout 600s;
proxy_send_timeout 600s;
proxy_read_timeout 600s;
keepalive_timeout 600s;
send_timeout 60s;
location / {
proxy_read_timeout 600;
charset utf-8;
root /usr/share/nginx/qdata;
try_files $uri $uri/ /index.html;
index index.html index.htm;
}
location /prod-api {# Forward qData APIs
add_header 'Access-Control-Allow-Methods' 'GET, POST,PUT,DELETE, OPTIONS';
proxy_read_timeout 6000;
rewrite ^/prod-api/(.*)$ /$1 break;
proxy_set_header X-Real-Ip $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_pass http://192.168.3.35:8081;
}
location /jmreport {# Forward reports
add_header 'Access-Control-Allow-Methods' 'GET, POST,PUT,DELETE, OPTIONS';
proxy_read_timeout 6000;
rewrite ^/prod-api/(.*)$ /$1 break;
proxy_set_header X-Real-Ip $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_pass http://192.168.3.35:8081;
}
location /drag {# Forward large-screen pages
add_header 'Access-Control-Allow-Methods' 'GET, POST,PUT,DELETE, OPTIONS';
proxy_read_timeout 6000;
rewrite ^/prod-api/(.*)$ /$1 break;
proxy_set_header X-Real-Ip $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_pass http://192.168.3.35:8081;
}
}5.5 Write the docker-compose.yml script file
sudo vi docker-compose.ymlFill in the following content and save the file.
version: "2.0"
services:
nginx:
privileged: true
image: nginx:1.21.5
restart: always
ports:
- "8084:8084" # qportal access port
volumes:
- ./www:/usr/share/nginx # Static file directory
- ./nginx.conf:/etc/nginx/nginx.conf:ro # Mount the Nginx configuration file
- ./http_vhost/:/etc/nginx/http_vhost/:ro # Mount website configuration files
logging:
driver: "json-file"
options:
max-size: "2000m"
max-file: "100"5.6 Start
docker-compose up -d6. ZooKeeper
6.1 Download the Docker Compose offline package
https://dlcdn.apache.org/zookeeper/zookeeper-3.8.4/apache-zookeeper-3.8.4-bin.tar.gz
Download apache-zookeeper-3.8.4-bin.tar.gz and upload it to the server.
6.2 Extract and copy the configuration file
sudo tar -zxvf apache-zookeeper-3.8.4-bin.tar.gz
# Rename
sudo mv apache-zookeeper-3.8.4-bin zookeeper
# Copy the configuration file
sudo cp -r zookeeper/conf/zoo_sample.cfg zookeeper/conf/zoo.cfg6.3 Edit the configuration file
sudo vi zookeeper/conf/zoo.cfg
# Change dataDir=/tmp/zookeeper to dataDir=./tmp/zookeeper6.4 Start
sudo ./zookeeper/bin/zkServer.sh start
# After startup succeeds, view the status
sudo ./zookeeper/bin/zkServer.sh status
# Normal output
# ZooKeeper JMX enabled by default
# Using config: /opt/ds2/zookeeper/bin/../conf/zoo.cfg
# Client port found: 2181. Client address: localhost. Client SSL: false.
# Mode: standalone
# If zkServer.sh reports "Error: JAVA_HOME is not set and java could not be found in PATH.", start this way
vi ./zookeeper/conf/java.env
# Add the following content
export JAVA_HOME=/opt/java/openjdk
# If zkServer.sh startup fails, check whether 8080 is occupied. If occupied, add admin.serverPort=8078 in conf/zoo.cfg.7. Hadoop
7.1 Write the docker-compose.yml script file
sudo vi docker-compose.ymlFill in the following content and save the file.
version: "3.8"
services:
namenode:
image: apache/hadoop:3.3.6
hostname: namenode
container_name: namenode
ports:
- "9870:9870" # NameNode Web UI
- "8020:8020" # HDFS RPC
- "9000:9000"
env_file: ./config
environment:
ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name"
command: ["hdfs", "namenode"]
networks:
hadoop:
ipv4_address: 172.32.1.2
datanode:
image: apache/hadoop:3.3.6
hostname: datanode
container_name: datanode
ports:
- "9866:9866"
env_file: ./config
command: ["hdfs", "datanode"]
networks:
hadoop:
ipv4_address: 172.32.1.3
volumes:
- /data/hadoop:/data/dfs # Mount data
resourcemanager:
image: apache/hadoop:3.3.6
hostname: resourcemanager
container_name: resourcemanager
ports:
- "8088:8088" # YARN Web UI
env_file: ./config
command: ["yarn", "resourcemanager"]
networks:
hadoop:
ipv4_address: 172.32.1.5
nodemanager:
image: apache/hadoop:3.3.6
hostname: nodemanager
container_name: nodemanager
env_file: ./config
command: ["yarn", "nodemanager"]
networks:
hadoop:
ipv4_address: 172.32.1.6
networks:
hadoop:
driver: bridge
ipam:
config:
- subnet: 172.32.1.0/247.2 Configuration file
sudo vi config
# Add the following content
CORE-SITE.XML_fs.default.name=hdfs://namenode
CORE-SITE.XML_fs.defaultFS=hdfs://namenode
HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020
HDFS-SITE.XML_dfs.replication=1
HDFS-SITE.XML_dfs.datanode.use.datanode.hostname=true
HDFS-SITE.XML_dfs.disk.balancer.enabled=true
HDFS-SITE.XML_dfs.storage.policy.hot.creation.fallbacks=ARCHIVE
HDFS-SITE.XML_dfs.datanode.data.dir=/data/dfs
MAPRED-SITE.XML_mapreduce.framework.name=yarn
MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager
YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false
YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600
YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false
YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=*
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=*
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings=
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false7.3 Start
docker-compose up -d7.4 Create base directories
docker exec -it namenode bash
hdfs dfs -mkdir -p /tmp/etl
hdfs dfs -chmod -R 777 /tmp/etl7.5 Edit hosts
# On the server that needs to connect to HDFS, for example the Spark server, edit hosts
sudo vi /etc/hosts
# Add the following content. Change 172.32.1.2 to the server private IP; for external HDFS access, use the public IP.
172.32.1.2 datanode8. MongoDB
8.1 Write the docker-compose.yml script file
sudo vi docker-compose.ymlFill in the following content and save the file.
version: "3.0"
services:
mongodb:
container_name: mongodb
restart: always
image: mongo:4.4
ports:
- 27017:27017
volumes:
- /data/mongoDB:/data/db # Mount data
environment:
MONGO_INITDB_ROOT_USERNAME: sjzt # Account
MONGO_INITDB_ROOT_PASSWORD: Desl9Y4eIQP1BHh7 # Password8.2 Start
docker-compose up -d