diff options
| -rwxr-xr-x | deploy_vm1.sh | 21 | ||||
| -rwxr-xr-x | deploy_vm2.sh | 22 | ||||
| -rw-r--r-- | docker-compose-app.yml | 68 | ||||
| -rw-r--r-- | docker-compose-db.yml | 19 | ||||
| -rw-r--r-- | elasticsearch/config/elasticsearch.yml | 400 | ||||
| -rwxr-xr-x | search-data-service/appconfig/elastic-search.properties | 2 |
6 files changed, 523 insertions, 9 deletions
diff --git a/deploy_vm1.sh b/deploy_vm1.sh index 09ddfff..19b243d 100755 --- a/deploy_vm1.sh +++ b/deploy_vm1.sh @@ -9,6 +9,9 @@ fi export RESOURCES_LOGS="/opt/aai/logroot/AAI-RESOURCES"; export TRAVERSAL_LOGS="/opt/aai/logroot/AAI-TRAVERSAL"; +export SEARCH_LOGS="/opt/aai/logroot/AAI-SEARCH"; +export DATA-ROUTER_LOGS="/opt/aai/logroot/AAI-DATA-ROUTER"; +export MODEL-LOADER_LOGS="/opt/aai/logroot/AAI-MODEL-LOADER"; if [ ! -d "$RESOURCES_LOGS" ]; then @@ -22,6 +25,24 @@ then mkdir -p $TRAVERSAL_LOGS; fi; +if [ ! -d "$SEARCH_LOGS" ]; +then + echo "Warning: Unable to find the volume directory $SEARCH_LOGS so creating it as regular directory"; + mkdir -p $SEARCH_LOGS; +fi; + +if [ ! -d "$DATA-ROUTER_LOGS" ]; +then + echo "Warning: Unable to find the volume directory $DATA-ROUTER_LOGS so creating it as regular directory"; + mkdir -p $DATA-ROUTER_LOGS; +fi; + +if [ ! -d "$MODEL-LOADER_LOGS" ]; +then + echo "Warning: Unable to find the volume directory $MODEL-LOADER_LOGS so creating it as regular directory"; + mkdir -p $MODEL-LOADER_LOGS; +fi; + export MTU=${MTU:-1500}; export DOCKER_REGISTRY="${DOCKER_REGISTRY:-localhost:5000}"; export HBASE_IMAGE="${HBASE_IMAGE:-wc9368/aai-hbase-1.2.3}"; diff --git a/deploy_vm2.sh b/deploy_vm2.sh index 305a2b2..d890528 100755 --- a/deploy_vm2.sh +++ b/deploy_vm2.sh @@ -9,6 +9,9 @@ fi export RESOURCES_LOGS="/opt/aai/logroot/AAI-RESOURCES"; export TRAVERSAL_LOGS="/opt/aai/logroot/AAI-TRAVERSAL"; +export SEARCH_LOGS="/opt/aai/logroot/AAI-SEARCH"; +export DATA-ROUTER_LOGS="/opt/aai/logroot/AAI-DATA-ROUTER"; +export MODEL-LOADER_LOGS="/opt/aai/logroot/AAI-MODEL-LOADER"; if [ ! -d "$RESOURCES_LOGS" ]; then @@ -22,6 +25,24 @@ then mkdir -p $TRAVERSAL_LOGS; fi; +if [ ! -d "$SEARCH_LOGS" ]; +then + echo "Warning: Unable to find the volume directory $SEARCH_LOGS so creating it as regular directory"; + mkdir -p $SEARCH_LOGS; +fi; + +if [ ! -d "$DATA-ROUTER_LOGS" ]; +then + echo "Warning: Unable to find the volume directory $DATA-ROUTER_LOGS so creating it as regular directory"; + mkdir -p $DATA-ROUTER_LOGS; +fi; + +if [ ! -d "$MODEL-LOADER_LOGS" ]; +then + echo "Warning: Unable to find the volume directory $MODEL-LOADER_LOGS so creating it as regular directory"; + mkdir -p $MODEL-LOADER_LOGS; +fi; + export MTU=${MTU:-1500}; export HBASE_IMAGE="${HBASE_IMAGE:-wc9368/aai-hbase-1.2.3}"; #export HBASE_IMAGE="${HBASE_IMAGE:-harisekhon/hbase}"; @@ -60,4 +81,3 @@ wait_for_container $HBASE_CONTAINER_NAME '^starting regionserver'; GREMLIN_CONTAINER_NAME=$($DOCKER_COMPOSE_CMD up -d aai.gremlinserver.simpledemo.openecomp.org 2>&1 | grep 'Creating' | awk '{ print $2; }' | head -1); wait_for_container $GREMLIN_CONTAINER_NAME 'Channel started at port 8182'; - diff --git a/docker-compose-app.yml b/docker-compose-app.yml index a149047..3ae78ce 100644 --- a/docker-compose-app.yml +++ b/docker-compose-app.yml @@ -9,9 +9,9 @@ services: - CHEF_BRANCH=master - CHEF_GIT_URL=http://gerrit.onap.org/r/aai - AAI_CORE_VERSION=1.1.0-SNAPSHOT - ports: + ports: - 8447:8447 - volumes: + volumes: - ${RESOURCES_LOGS}:/opt/aai/logroot/AAI-RES logging: driver: "json-file" @@ -27,9 +27,9 @@ services: - CHEF_BRANCH=master - CHEF_GIT_URL=http://gerrit.onap.org/r/aai - AAI_CORE_VERSION=1.1.0-SNAPSHOT - volumes: + volumes: - ${TRAVERSAL_LOGS}:/opt/aai/logroot/AAI-GQ - ports: + ports: - 8446:8446 logging: driver: "json-file" @@ -39,9 +39,9 @@ services: aai.api.simpledemo.openecomp.org: image: ${AAI_HAPROXY_IMAGE} hostname: aai.api.simpledemo.openecomp.org - ports: + ports: - 8443:8443 - links: + links: - aai-resources.api.simpledemo.openecomp.org - aai-traversal.api.simpledemo.openecomp.org volumes: @@ -51,8 +51,62 @@ services: options: max-size: "30m" max-file: "5" + + aai.searchservice.simpledemo.openecomp.org: + image: ${DOCKER_REGISTRY}/openecomp/search-data-service + hostname: aai.searchservice.simpledemo.openecomp.org + volumes: + - ${SEARCH_LOGS}:/logs + - /opt/app/search-data-service/appconfig:/opt/app/search-data-service/config + ports: + - 9509:9509 + environment: + - CONFIG_HOME=/opt/app/search-data-service/config/ + - KEY_STORE_PASSWORD=OBF:1y0q1uvc1uum1uvg1pil1pjl1uuq1uvk1uuu1y10 + - KEY_MANAGER_PASSWORD=OBF:1y0q1uvc1uum1uvg1pil1pjl1uuq1uvk1uuu1y10 + logging: + driver: "json-file" + options: + max-size: "30m" + max-file: "5" + + datarouter: + image: ${DOCKER_REGISTRY}/openecomp/datarouter-service + hostname: datarouter + volumes: + - ${DATA-ROUTER_LOGS}:/logs + - /opt/app/datarouter-service/appconfig:/opt/app/data-router/config + - /opt/app/datarouter-service/dynamic:/opt/app/data-router/dynamic + ports: + - 9502:9502 + environment: + - CONFIG_HOME=/opt/app/data-router/config/ + - KEY_STORE_PASSWORD=OBF:1i9a1u2a1unz1lr61wn51wn11lss1unz1u301i6o + - KEY_MANAGER_PASSWORD=OBF:1y0q1uvc1uum1uvg1pil1pjl1uuq1uvk1uuu1y10 + - DYNAMIC_ROUTES=/opt/app/data-router/dynamic/routes + - SERVICE_BEANS=/opt/app/data-router/dynamic/conf + logging: + driver: "json-file" + options: + max-size: "30m" + max-file: "5" + + model-loader: + image: ${DOCKER_REGISTRY}/openecomp/model-loader + hostname: model-loader + volumes: + - ${MODEL-LOADER_LOGS}:/logs + - /opt/app/model-loader/appconfig:/opt/app/model-loader/config + environment: + - CONFIG_HOME=/opt/app/model-loader/config/ + logging: + driver: "json-file" + options: + max-size: "30m" + max-file: "5" + networks: - default: + default: driver: bridge driver_opts: com.docker.network.driver.mtu: ${MTU} diff --git a/docker-compose-db.yml b/docker-compose-db.yml index 7f55858..da29fdd 100644 --- a/docker-compose-db.yml +++ b/docker-compose-db.yml @@ -24,8 +24,27 @@ services: options: max-size: "30m" max-file: "5" + + aai.elasticsearch.simpledemo.openecomp.org: + image: elasticsearch:2.4.1 + hostname: aai.elasticsearch.simpledemo.openecomp.org + container_name: elasticsearch + ports: + - "9200:9200" + volumes: + - /opt/app/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml + - es-data:/usr/share/elasticsearch/data + logging: + driver: "json-file" + options: + max-size: "30m" + max-file: "5" + networks: default: driver: bridge driver_opts: com.docker.network.driver.mtu: ${MTU} + +volumes: + es-data: diff --git a/elasticsearch/config/elasticsearch.yml b/elasticsearch/config/elasticsearch.yml new file mode 100644 index 0000000..21e29df --- /dev/null +++ b/elasticsearch/config/elasticsearch.yml @@ -0,0 +1,400 @@ +##################### Elasticsearch Configuration Example ##################### + +# This file contains an overview of various configuration settings, +# targeted at operations staff. Application developers should +# consult the guide at <http://elasticsearch.org/guide>. +# +# The installation procedure is covered at +# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html>. +# +# Elasticsearch comes with reasonable defaults for most settings, +# so you can try it out without bothering with configuration. +# +# Most of the time, these defaults are just fine for running a production +# cluster. If you're fine-tuning your cluster, or wondering about the +# effect of certain configuration option, please _do ask_ on the +# mailing list or IRC channel [http://elasticsearch.org/community]. + +# Any element in the configuration can be replaced with environment variables +# by placing them in ${...} notation. For example: +# +# node.rack: ${RACK_ENV_VAR} + +# For information on supported formats and syntax for the config file, see +# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/setup-configuration.html> +################################### Cluster ################################### + +# Cluster name identifies your cluster for auto-discovery. If you're running +# multiple clusters on the same network, make sure you're using unique names. +# +# cluster.name: elasticsearch + +cluster.name: ES_AAI + +#################################### Node ##################################### + +node.name: ES_ONAP +node.master: true +node.data: true + + +# Use the Cluster Health API [http://localhost:9200/_cluster/health], the +# Node Info API [http://localhost:9200/_nodes] or GUI tools +# such as <http://www.elasticsearch.org/overview/marvel/>, +# <http://github.com/karmi/elasticsearch-paramedic>, +# <http://github.com/lukas-vlcek/bigdesk> and +# <http://mobz.github.com/elasticsearch-head> to inspect the cluster state. + +# By default, multiple nodes are allowed to start from the same installation location +# to disable it, set the following: + +node.max_local_storage_nodes: 1 + + +#################################### Index #################################### +# You can set a number of options (such as shard/replica options, mapping +# or analyzer definitions, translog settings, ...) for indices globally, +# in this file. +# +# Note, that it makes more sense to configure index settings specifically for +# a certain index, either when creating it or by using the index templates API. +# +# See <http://elasticsearch.org/guide/en/elasticsearch/reference/current/index-modules.html> and +# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/indices-create-index.html> +# for more information. + +# Set the number of shards (splits) of an index (5 by default): + +index.number_of_shards: 5 + +# Set the number of replicas (additional copies) of an index (1 by default): + +index.number_of_replicas: 1 + +# These settings directly affect the performance of index and search operations +# in your cluster. Assuming you have enough machines to hold shards and +# replicas, the rule of thumb is: +# +# 1. Having more *shards* enhances the _indexing_ performance and allows to +# _distribute_ a big index across machines. +# 2. Having more *replicas* enhances the _search_ performance and improves the +# cluster _availability_. +# +# The "number_of_shards" is a one-time setting for an index. +# +# The "number_of_replicas" can be increased or decreased anytime, +# by using the Index Update Settings API. +# +# Elasticsearch takes care about load balancing, relocating, gathering the +# results from nodes, etc. Experiment with different settings to fine-tune +# your setup. + +# Use the Index Status API (<http://localhost:9200/A/_status>) to inspect +# the index status. + + +#################################### Paths #################################### + +# Path to directory containing configuration (this file and logging.yml): +#path.conf: /opt/app/elasticsearch/config + +# Path to directory where to store index data allocated for this node. +# Use swm auto link to redirect the data directory if necessary. + +#path.data: /opt/app/elasticsearch/data + +# path.data: /path/to/data1,/path/to/data2 + +# path.work: /path/to/work + +#path.logs: /opt/app/elasticsearch/logs + +#path.plugins: /opt/app/elasticsearch/plugins + + +#################################### Plugin ################################### + +# If a plugin listed here is not installed for current node, the node will not start. +# +# plugin.mandatory: mapper-attachments,lang-groovy + + +################################### Memory #################################### + +# Elasticsearch performs poorly when JVM starts swapping: you should ensure that +# it _never_ swaps. +# +# Set this property to true to lock the memory: default is true + +bootstrap.mlockall: true + +# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set +# to the same value, and that the machine has enough memory to allocate +# for Elasticsearch, leaving enough memory for the operating system itself. +# +# You should also make sure that the Elasticsearch process is allowed to lock +# the memory, eg. by using `ulimit -l unlimited`. + + +############################## Network And HTTP ############################### +# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens +# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node +# communication. (the range means that if the port is busy, it will automatically +# try the next port). + +# Set the bind address specifically (IPv4 or IPv6): +network.bind_host: 0.0.0.0 + +# Set the address other nodes will use to communicate with this node. If not +# set, it is automatically derived. It must point to an actual IP address. + +# network.publish_host: 0.0.0.0 + +# Set both 'bind_host' and 'publish_host': +# network.host: 192.168.0.1 + + +# Set a custom port for the node to node communication (9300 by default): +transport.tcp.port: 8443 + +# Enable compression for all communication between nodes (disabled by default): +transport.tcp.compress: false + +# Set a custom port to listen for HTTP traffic: +# http.port: 9200 +http.port: 9200 + +# Set a custom allowed content length: +# http.max_content_length: 100mb +http.max_content_length: 100mb + +# Disable HTTP completely: +# http.enabled: false +http.enabled: true + +# This is specifically useful for permitting which front end Kibana Url's are permitted to access elastic search. +http.cors.enabled: false +http.cors.allow-origin: "/.*/" +http.cors.allow-headers: X-Requested-With, Content-Type, Content-Length +http.cors.allow-credentials: false +################################### Gateway ################################### + +# The gateway allows for persisting the cluster state between full cluster +# restarts. Every change to the state (such as adding an index) will be stored +# in the gateway, and when the cluster starts up for the first time, +# it will read its state from the gateway. +# There are several types of gateway implementations. For more information, see +# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-gateway.html>. + +# The default gateway type is the "local" gateway (recommended): +# +#gateway.type: local +#gateway.type: local + +# Settings below control how and when to start the initial recovery process on +# a full cluster restart (to reuse as much local data as possible when using shared +# gateway). + +# Allow recovery process after N nodes in a cluster are up: +# +# gateway.recover_after_nodes: 1 +gateway.recover_after_nodes: 1 + +# Set the timeout to initiate the recovery process, once the N nodes +# from previous setting are up (accepts time value): +# +#gateway.recover_after_time: 5m +gateway.recover_after_time: 5m + +# Set how many nodes are expected in this cluster. Once these N nodes +# are up (and recover_after_nodes is met), begin recovery process immediately +# (without waiting for recover_after_time to expire): +# +# gateway.expected_nodes: 2 +gateway.expected_nodes: 2 + +############################# Recovery Throttling ############################# + +# These settings allow to control the process of shards allocation between +# nodes during initial recovery, replica allocation, rebalancing, +# or when adding and removing nodes. + +# Set the number of concurrent recoveries happening on a node: +# +# 1. During the initial recovery +# +# cluster.routing.allocation.node_initial_primaries_recoveries: 4 +# +# 2. During adding/removing nodes, rebalancing, etc +# +# cluster.routing.allocation.node_concurrent_recoveries: 2 + +# Set to throttle throughput when recovering (eg. 100mb, by default 20mb): +# indices.recovery.max_bytes_per_sec: 20mb +indices.recovery.max_bytes_per_sec: 20mb + +# Set to limit the number of open concurrent streams when +# recovering a shard from a peer: +# +# indices.recovery.concurrent_streams: 5 +indices.recovery.concurrent_streams: 5 + +################################## Discovery ################################## + +# Discovery infrastructure ensures nodes can be found within a cluster +# and master node is elected. Multicast discovery is the default. + +# Set to ensure a node sees N other master eligible nodes to be considered +# operational within the cluster. Its recommended to set it to a higher value +# than 1 when running more than 2 nodes in the cluster. +# +discovery.zen.minimum_master_nodes: 1 + +# Set the time to wait for ping responses from other nodes when discovering. +# Set this option to a higher value on a slow or congested network +# to minimize discovery failures: +# +# discovery.zen.ping.timeout: 3s +discovery.zen.ping.timeout: + +# For more information, see +# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-zen.html> + +# Unicast discovery allows to explicitly control which nodes will be used +# to discover the cluster. It can be used when multicast is not present, +# or to restrict the cluster communication-wise. +# +# 1. Disable multicast discovery (enabled by default): +# discovery.zen.ping.multicast.enabled: false +discovery.zen.ping.multicast.enabled: false + + +# 2. Configure an initial list of master nodes in the cluster +# to perform discovery when new nodes (master or data) are started: +# +# discovery.zen.ping.unicast.hosts: ["host1", "host2:port"] +discovery.zen.ping.unicast.hosts: ["0.0.0.0"] + +# EC2 discovery allows to use AWS EC2 API in order to perform discovery. +# +# You have to install the cloud-aws plugin for enabling the EC2 discovery. +# +# For more information, see +# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-ec2.html> +# +# +# See <http://elasticsearch.org/tutorials/elasticsearch-on-ec2/> +# for a step-by-step tutorial. + +# GCE discovery allows to use Google Compute Engine API in order to perform discovery. +# +# You have to install the cloud-gce plugin for enabling the GCE discovery. +# +# For more information, see <https://github.com/elasticsearch/elasticsearch-cloud-gce>. + +# Azure discovery allows to use Azure API in order to perform discovery. +# +# You have to install the cloud-azure plugin for enabling the Azure discovery. +# +# For more information, see <https://github.com/elasticsearch/elasticsearch-cloud-azure>. + +################################## Slow Log ################################## + +# Shard level query and fetch threshold logging. + +#index.search.slowlog.threshold.query.warn: 10s +#index.search.slowlog.threshold.query.info: 5s +#index.search.slowlog.threshold.query.debug: 2s +#index.search.slowlog.threshold.query.trace: 500ms + +#index.search.slowlog.threshold.fetch.warn: 1s +#index.search.slowlog.threshold.fetch.info: 800ms +#index.search.slowlog.threshold.fetch.debug: 500ms +#index.search.slowlog.threshold.fetch.trace: 200ms + +#index.indexing.slowlog.threshold.index.warn: 10s +#index.indexing.slowlog.threshold.index.info: 5s +#index.indexing.slowlog.threshold.index.debug: 2s +#index.indexing.slowlog.threshold.index.trace: 500ms + +################################## GC Logging ################################ + +#monitor.jvm.gc.young.warn: 1000ms +#monitor.jvm.gc.young.info: 700ms +#monitor.jvm.gc.young.debug: 400ms + +#monitor.jvm.gc.old.warn: 10s +#monitor.jvm.gc.old.info: 5s +#monitor.jvm.gc.old.debug: 2s + +############################################################################################# +### SEARCH GUARD SSL # +### Configuration # +############################################################################################### +## Uncomment all lines below prefixed with #X# (globally remove #X#) for searchguard +## +############################################################################################### +### Transport layer SSL # +### # +############################################################################################### +### Enable or disable node-to-node ssl encryption (default: true) +#X#searchguard.ssl.transport.enable_openssl_if_available: true +#X#searchguard.ssl.transport.enabled: true +### JKS or PKCS12 (default: JKS) +#X#searchguard.ssl.transport.keystore_type: JKS +### Relative path to the keystore file (mandatory, this stores the server certificates), must be placed under the config/ dir +#X#searchguard.ssl.transport.keystore_filepath: /some/path +### Alias name (default: first alias which could be found) +###searchguard.ssl.transport.keystore_alias: localhost +### Keystore password (default: changeit) +#X#searchguard.ssl.transport.keystore_password: changeit +## +### JKS or PKCS12 (default: JKS) +#X#searchguard.ssl.transport.truststore_type: JKS +### Relative path to the truststore file (mandatory, this stores the client/root certificates), must be placed under the config/ dir +#X#searchguard.ssl.transport.truststore_filepath: truststore.jks +### Alias name (default: first alias which could be found) +###searchguard.ssl.transport.truststore_alias: my_alias +### Truststore password (default: changeit) +#X#searchguard.ssl.transport.truststore_password: changeit +### Enforce hostname verification (default: true) +###searchguard.ssl.transport.enforce_hostname_verification: true +### If hostname verification specify if hostname should be resolved (default: true) +###searchguard.ssl.transport.resolve_hostname: true +### Use native Open SSL instead of JDK SSL if available (default: true) +###searchguard.ssl.transport.enable_openssl_if_available: false +## +############################################################################################### +### HTTP/REST layer SSL # +### # +############################################################################################### +### Enable or disable rest layer security - https, (default: false) +#X#searchguard.ssl.http.enable_openssl_if_available: true +#X#searchguard.ssl.http.enabled: true +### JKS or PKCS12 (default: JKS) +#X#searchguard.ssl.http.keystore_type: JKS +### Relative path to the keystore file (this stores the server certificates), must be placed under the config/ dir +#X#searchguard.ssl.http.keystore_filepath: /keystore/path +### Alias name (default: first alias which could be found) +###searchguard.ssl.http.keystore_alias: my_alias +### Keystore password (default: changeit) +#X#searchguard.ssl.http.keystore_password: changeit +### Do the clients (typically the browser or the proxy) have to authenticate themself to the http server, default is OPTIONAL +### To enforce authentication use REQUIRE, to completely disable client certificates use NONE +###searchguard.ssl.http.clientauth_mode: REQUIRE +### JKS or PKCS12 (default: JKS) +#X#searchguard.ssl.http.truststore_type: JKS +### Relative path to the truststore file (this stores the client certificates), must be placed under the config/ dir +#X#searchguard.ssl.http.truststore_filepath: truststore.jks +### Alias name (default: first alias which could be found) +###searchguard.ssl.http.truststore_alias: my_alias +### Truststore password (default: changeit) +#X#searchguard.ssl.http.truststore_password: changeit +### Use native Open SSL instead of JDK SSL if available (default: true) +###searchguard.ssl.http.enable_openssl_if_available: false + +##################################################### +##### Security manager - Searchguard Configuration +##################################################### +#X#security.manager.enabled: false +#X#searchguard.authcz.admin_dn: diff --git a/search-data-service/appconfig/elastic-search.properties b/search-data-service/appconfig/elastic-search.properties index 7b442b0..341418d 100755 --- a/search-data-service/appconfig/elastic-search.properties +++ b/search-data-service/appconfig/elastic-search.properties @@ -1,5 +1,5 @@ # ElasticSearch Configuration -es.cluster-name= +es.cluster-name=ES_AAI es.ip-address= es.http-port=9200 |