From afe83ec25c63de80f231111cde46c8b0b4744b6e Mon Sep 17 00:00:00 2001 From: Pratik Raj Date: Sat, 6 Jun 2020 21:59:32 +0530 Subject: optimize size and time using "--no-cache-dir" Using "--no-cache-dir" flag in pip install ,make sure dowloaded packages by pip don't cached on system . This is a best practise which make sure to fetch ftom repo instead of using local cached one . Further , in case of Docker Containers , by restricing caching , we can reduce image size. In term of stats , it depends upon the number of python packages multiplied by their respective size . e.g for heavy packages with a lot of dependencies it reduce a lot by don't caching pip packages. Further , more detail information can be found at https://medium.com/sciforce/strategies-of-docker-images-optimization-2ca9cc5719b6 Issue-ID: INT-1616 Signed-off-by: Pratik Raj Change-Id: Id3e28faf35f36258362323b4a96bcf0f3f95726b Signed-off-by: mrichomme --- S3Ptools/locust_Grafana.sh | 2 +- deployment/aks/util/create_openstack_cli.sh | 2 +- test/hpa_automation/tosca/hpa_automation.py | 44 +++++++++++----------- .../mr-sim/Dockerfile | 2 +- .../datafilecollector-testharness/mr-sim/setup.sh | 2 +- .../engine/configure-modules.sh | 2 +- test/mocks/prov-mns-provider/Dockerfile | 2 +- test/vcpe/bin/setup.sh | 2 +- test/vcpe/tox.ini | 4 +- test/vcpe/vcpecommon.py | 4 +- 10 files changed, 33 insertions(+), 33 deletions(-) diff --git a/S3Ptools/locust_Grafana.sh b/S3Ptools/locust_Grafana.sh index 5fa6fdcdd..5168f0e92 100644 --- a/S3Ptools/locust_Grafana.sh +++ b/S3Ptools/locust_Grafana.sh @@ -1,7 +1,7 @@ #!/bin/bash #grafana install for the use of locust # localgosh:80(grafana) & localhost:81 -pip install docker==3.1.4 +pip install --no-cache-dir docker==3.1.4 git clone https://github.com/kamon-io/docker-grafana-graphite.git cd docker-grafana-graphite make up diff --git a/deployment/aks/util/create_openstack_cli.sh b/deployment/aks/util/create_openstack_cli.sh index 8f3f331b3..3e69ac403 100755 --- a/deployment/aks/util/create_openstack_cli.sh +++ b/deployment/aks/util/create_openstack_cli.sh @@ -46,7 +46,7 @@ spec: apk add musl-dev && \ apk add libffi-dev && \ apk add openssl-dev && \ - pip3 install python-openstackclient && \ + pip3 install --no-cache-dir python-openstackclient && \ sh -c 'echo ". /openstack/openstack_rc" >> /root/.profile; while true; do sleep 60; done;' restartPolicy: Never volumes: diff --git a/test/hpa_automation/tosca/hpa_automation.py b/test/hpa_automation/tosca/hpa_automation.py index 93027237c..ac6a161d2 100755 --- a/test/hpa_automation/tosca/hpa_automation.py +++ b/test/hpa_automation/tosca/hpa_automation.py @@ -3,7 +3,7 @@ #Prerequisites for machine to run this #Put in required parameters in hpa_automation_config.json #Install python-pip (apt install python-pip) -#Install python mysql.connector (pip install mysql-connector-python) +#Install python mysql.connector (pip install --no-cache-dir mysql-connector-python) #Install ONAP CLI #Must have connectivity to the ONAP, a k8s vm already running is recommended #Create Preload File, the script will modify the parameters required from serivce model, service instance @@ -223,16 +223,16 @@ def create_customer(parameters): def add_customer_subscription(parameters): subscription_check = 0 for cloud_region, cloud_region_values in (parameters["cloud_region_data"]).iteritems(): - if subscription_check == 0 : - subscription_string = "oclip subscription-create -x {} -c {} -z {} -e {} -y {} -r {} -m {} -u {} -p {}".format(\ - parameters["customer_name"], cloud_region_values.get("tenant-id"), parameters["cloud-owner"], parameters["service_name"],\ - cloud_region_values.get("default-tenant"), cloud_region, parameters["aai_url"], parameters["aai_username"], parameters["aai_password"] ) - else: - subscription_string = "oclip subscription-cloud-add -x {} -c {} -z {} -e {} -y {} -r {} -m {} -u {} -p {}".format(\ - parameters["customer_name"], cloud_region_values.get("tenant-id"), parameters["cloud-owner"], parameters["service_name"],\ - cloud_region_values.get("default-tenant"), cloud_region, parameters["aai_url"], parameters["aai_username"], parameters["aai_password"] ) - os.system(subscription_string) - subscription_check+=1 + if subscription_check == 0 : + subscription_string = "oclip subscription-create -x {} -c {} -z {} -e {} -y {} -r {} -m {} -u {} -p {}".format(\ + parameters["customer_name"], cloud_region_values.get("tenant-id"), parameters["cloud-owner"], parameters["service_name"],\ + cloud_region_values.get("default-tenant"), cloud_region, parameters["aai_url"], parameters["aai_username"], parameters["aai_password"] ) + else: + subscription_string = "oclip subscription-cloud-add -x {} -c {} -z {} -e {} -y {} -r {} -m {} -u {} -p {}".format(\ + parameters["customer_name"], cloud_region_values.get("tenant-id"), parameters["cloud-owner"], parameters["service_name"],\ + cloud_region_values.get("default-tenant"), cloud_region, parameters["aai_url"], parameters["aai_username"], parameters["aai_password"] ) + os.system(subscription_string) + subscription_check+=1 def register_vnfm_helper(vnfm_key, values, parameters): #Create vnfm @@ -249,7 +249,7 @@ def register_vnfm(parameters): for vnfm_key, vnfm_values in vnfm_params.iteritems(): register_vnfm_helper(vnfm_key, vnfm_values, parameters) -def add_policy_models(parameters): +def add_policy_models(): mydb = mysql.connector.connect( host="policydb", user="policy_user", @@ -361,15 +361,15 @@ def add_policy_models(parameters): def add_policies(parameters): #Loop through policy, put in resource_model_name and create policies for policy in os.listdir(parameters["policy_directory"]): - policy_name = "{}.{}".format(parameters["policy_scope"], os.path.splitext(policy)[0]) - policy_file = (os.path.join(parameters["policy_directory"], policy)) - #Create policy - os.system("oclip policy-create-outdated -m {} -u {} -p {} -x {} -S {} -T {} -o {} -b $(cat {})".format(parameters["policy_url"],\ - parameters["policy_username"], parameters["policy_password"], policy_name, parameters["policy_scope"], \ - parameters["policy_config_type"], parameters["policy_onapName"], policy_file)) - - #Push policy - os.system("oclip policy-push-outdated -m {} -u {} -p {} -x {} -b {} -c {}".format(parameters["policy_url"], \ + policy_name = "{}.{}".format(parameters["policy_scope"], os.path.splitext(policy)[0]) + policy_file = (os.path.join(parameters["policy_directory"], policy)) + # Create policy + os.system("oclip policy-create-outdated -m {} -u {} -p {} -x {} -S {} -T {} -o {} -b $(cat {})".format(parameters["policy_url"],\ + parameters["policy_username"], parameters["policy_password"], policy_name, parameters["policy_scope"], \ + parameters["policy_config_type"], parameters["policy_onapName"], policy_file)) + + # Push policy + os.system("oclip policy-push-outdated -m {} -u {} -p {} -x {} -b {} -c {}".format(parameters["policy_url"], \ parameters["policy_username"], parameters["policy_password"], policy_name, parameters["policy_config_type"],\ parameters["policy_pdp_group"])) @@ -521,7 +521,7 @@ else: print(ns_out) # 6.add_policies function not currently working, using curl commands -add_policy_models(parameters) +add_policy_models() add_policies(parameters) # 7. VFC part diff --git a/test/mocks/datafilecollector-testharness/mr-sim/Dockerfile b/test/mocks/datafilecollector-testharness/mr-sim/Dockerfile index c54713e7f..bb21a5276 100755 --- a/test/mocks/datafilecollector-testharness/mr-sim/Dockerfile +++ b/test/mocks/datafilecollector-testharness/mr-sim/Dockerfile @@ -4,6 +4,6 @@ COPY . /app WORKDIR /app -RUN pip install -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt EXPOSE 2222 2223 diff --git a/test/mocks/datafilecollector-testharness/mr-sim/setup.sh b/test/mocks/datafilecollector-testharness/mr-sim/setup.sh index 2c941361f..e6f50b25f 100755 --- a/test/mocks/datafilecollector-testharness/mr-sim/setup.sh +++ b/test/mocks/datafilecollector-testharness/mr-sim/setup.sh @@ -10,4 +10,4 @@ else virtualenv -p python3 .env fi -source .env/bin/activate && pip3 install -r requirements.txt +source .env/bin/activate && pip3 install --no-cache-dir -r requirements.txt diff --git a/test/mocks/netconf-pnp-simulator/engine/configure-modules.sh b/test/mocks/netconf-pnp-simulator/engine/configure-modules.sh index d40918f31..4e1d17c25 100755 --- a/test/mocks/netconf-pnp-simulator/engine/configure-modules.sh +++ b/test/mocks/netconf-pnp-simulator/engine/configure-modules.sh @@ -81,7 +81,7 @@ create_python_venv() cd $env_dir # shellcheck disable=SC1091 . ./bin/activate - pip install --requirement "$dir"/requirements.txt + pip install --no-cache-dir --requirement "$dir"/requirements.txt ) 1>&2 echo $env_dir } diff --git a/test/mocks/prov-mns-provider/Dockerfile b/test/mocks/prov-mns-provider/Dockerfile index 748ad4825..fef09b239 100644 --- a/test/mocks/prov-mns-provider/Dockerfile +++ b/test/mocks/prov-mns-provider/Dockerfile @@ -4,7 +4,7 @@ WORKDIR /app COPY src/requirements.txt ./ -RUN pip install -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt COPY src /app diff --git a/test/vcpe/bin/setup.sh b/test/vcpe/bin/setup.sh index ad0e78c74..30c25cb62 100755 --- a/test/vcpe/bin/setup.sh +++ b/test/vcpe/bin/setup.sh @@ -36,4 +36,4 @@ pip_setup_path="$(dirname $0)/../" curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py python get-pip.py -pip install -I ${pip_setup_path} +pip install --no-cache-dir -I ${pip_setup_path} diff --git a/test/vcpe/tox.ini b/test/vcpe/tox.ini index 25785a67b..8da9312f8 100644 --- a/test/vcpe/tox.ini +++ b/test/vcpe/tox.ini @@ -1,6 +1,6 @@ # tox (https://tox.readthedocs.io/) is a tool for running tests # in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" +# test suite on all supported python versions. To use it, "pip install --no-cache-dir tox" # and then run "tox" from this directory. [tox] @@ -10,6 +10,6 @@ envlist = pytest deps = pytest [testenv:pytest] -basepython = python2 +basepython = python3 commands = pytest diff --git a/test/vcpe/vcpecommon.py b/test/vcpe/vcpecommon.py index 971082c80..0e02987ff 100755 --- a/test/vcpe/vcpecommon.py +++ b/test/vcpe/vcpecommon.py @@ -10,7 +10,7 @@ import sys import ipaddress import mysql.connector import requests -import commands +import subprocess import time import yaml from novaclient import client as openstackclient @@ -222,7 +222,7 @@ class VcpeCommon: self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name)) if not self.oom_mode: cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid) - ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd)) + ret = subprocess.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd)) self.logger.debug('%s', ret) else: print('To add vGMUX vserver info to AAI, do the following:') -- cgit 1.2.3-korg