path: root/asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/PopulateComponentCache.java

/*-
 * ============LICENSE_START=======================================================
 * SDC
 * ================================================================================
 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
 * ================================================================================
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * ============LICENSE_END=========================================================
 */

package org.openecomp.sdc.asdctool.impl;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
import org.openecomp.sdc.be.dao.cassandra.ComponentCassandraDao;
import org.openecomp.sdc.be.dao.neo4j.GraphPropertiesDictionary;
import org.openecomp.sdc.be.dao.titan.TitanOperationStatus;
import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
import org.openecomp.sdc.be.model.Component;
import org.openecomp.sdc.be.model.Product;
import org.openecomp.sdc.be.model.Resource;
import org.openecomp.sdc.be.model.Service;
import org.openecomp.sdc.be.model.cache.ComponentCache;
import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
import org.openecomp.sdc.be.model.operations.impl.ProductOperation;
import org.openecomp.sdc.be.model.operations.impl.ResourceOperation;
import org.openecomp.sdc.be.model.operations.impl.ServiceOperation;
import org.openecomp.sdc.be.resources.data.ComponentCacheData;
import org.openecomp.sdc.be.resources.data.ESArtifactData;
import org.openecomp.sdc.common.util.SerializationUtils;
import org.openecomp.sdc.common.util.ZipUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanVertex;

import fj.data.Either;

/**
 * Created by esofer on 9/1/2016.
 */
public class PopulateComponentCache {

	private static Logger log = LoggerFactory.getLogger(PopulateComponentCache.class.getName());

	@Autowired
	protected ComponentCassandraDao componentCassandraDao;

	@Autowired
	protected ResourceOperation resourceOperation;

	@Autowired
	protected ServiceOperation serviceOperation;

	@Autowired
	protected ProductOperation productOperation;

	@Autowired
	protected ComponentCache componentCache;

	private void exit(String stage, int i) {
		log.error("Failed on " + stage);
		System.exit(i);

	}

	public void populateCache() {
		populateCache(ComponentTypeEnum.RESOURCE);
		populateCache(ComponentTypeEnum.SERVICE);
		populateCache(ComponentTypeEnum.PRODUCT);
	}

	private void populateCache(ComponentTypeEnum componentTypeEnum) {

		List<String> list = new ArrayList<>();
		Either<TitanGraph, TitanOperationStatus> graph = resourceOperation.getTitanGenericDao().getGraph();
		TitanGraph titanGraph = graph.left().value();
		Iterable vertices = titanGraph.query()
				.has(GraphPropertiesDictionary.LABEL.getProperty(), componentTypeEnum.name().toLowerCase()).vertices();

		Iterator iterator = vertices.iterator();
		while (iterator.hasNext()) {
			TitanVertex vertex = (TitanVertex) iterator.next();

			// VertexProperty<Object> state =
			// vertex.property(GraphPropertiesDictionary.STATE.getProperty());
			// String stateValue = (String)state.value();

			// if (false ==
			// stateValue.equalsIgnoreCase(LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT.name())
			// ) {
			VertexProperty<Object> uid = vertex.property(GraphPropertiesDictionary.UNIQUE_ID.getProperty());
			String uidValue = (String) uid.value();

			list.add(uidValue);
			// }
		}

		int counter = 0;
		for (String componentUid : list) {

			long time1 = System.currentTimeMillis();

			/////////////////////////////////////////////////////////////////////////////////////
			// Pay attention. The component is fetched from the cache in case it
			///////////////////////////////////////////////////////////////////////////////////// is
			///////////////////////////////////////////////////////////////////////////////////// already
			///////////////////////////////////////////////////////////////////////////////////// there.
			/////////////////////////////////////////////////////////////////////////////////////
			Component component = null;
			switch (componentTypeEnum) {
			case RESOURCE:
				Either<Resource, StorageOperationStatus> resourceRes = resourceOperation.getComponent(componentUid,
						false);
				if (resourceRes.isRight()) {
					exit("get resource", 1);
				}
				component = resourceRes.left().value();
				break;
			case SERVICE:
				Either<Service, StorageOperationStatus> serviceRes = serviceOperation.getComponent(componentUid, false);
				if (serviceRes.isRight()) {
					exit("get service", 1);
				}
				component = serviceRes.left().value();
				break;
			case PRODUCT:
				Either<Product, StorageOperationStatus> productRes = productOperation.getComponent(componentUid, false);
				if (productRes.isRight()) {
					exit("get product", 1);
				}
				component = productRes.left().value();
				break;
			default:
				break;
			}

			if (component == null) {
				exit("get component", 1);
			}

			long time2 = System.currentTimeMillis();
			// System.out.println("fetch resource " + resource.getName());
			// System.out.println("fetch resource time is " + (time2 - time1) +
			// " ms");

			boolean setComponent = componentCache.setComponent(component, componentTypeEnum.getNodeType());
			if (setComponent) {
				counter++;
			}

			/*
			 * Either<byte[], Boolean> valueRes =
			 * SerializationUtils.serializeExt(component);
			 * 
			 * if (valueRes.isRight()) { exit("serialize component " +
			 * component.getName(), 2); } byte[] value =
			 * valueRes.left().value(); log.info("byte[] size is " +
			 * value.length); //System.out.println("byte[] size is " +
			 * value.length);
			 * 
			 * byte[] zipped = null; try { zipped = ZipUtil.zipBytes(value);
			 * //System.out.println("byte[] size after zip is " +
			 * zipped.length);
			 * 
			 * ComponentCacheData componentCacheData = new ComponentCacheData();
			 * componentCacheData.setDataAsArray(zipped);
			 * componentCacheData.setIsZipped(true);
			 * componentCacheData.setId(componentUid);
			 * componentCacheData.setModificationTime(new
			 * Date(component.getLastUpdateDate()));
			 * componentCacheData.setType(component.getComponentType().name().
			 * toLowerCase());
			 * 
			 * long averageInsertTimeInMilli =
			 * writeResourceToCassandraComponent(componentCacheData); log.
			 * info("After adding component {} to cassandra. Insert time is {} ms."
			 * , componentUid, averageInsertTimeInMilli);
			 * 
			 * } catch (IOException e) { // TODO Auto-generated catch block
			 * e.printStackTrace(); }
			 */

		}

		log.debug("The number of saved components of type {} is {}. Total size is {}", componentTypeEnum, counter,
				list.size());

	}

	private long writeResourceToCassandraComponent(ComponentCacheData componentCacheData) {

		long startTime = System.currentTimeMillis();

		// call to cassandra read
		CassandraOperationStatus saveArtifact = componentCassandraDao.saveComponent(componentCacheData);
		if (saveArtifact != CassandraOperationStatus.OK) {
			exit("writeResourceToCassandra", 3);
		}

		long endTime = System.currentTimeMillis();

		return (endTime - startTime);
	}

	private void deserializeByThreads(List<ESArtifactData> list, ExecutorService executor, int threadNumber) {

		long fullSearchStart = System.currentTimeMillis();
		// for (int k =0; k < parts; k++) {

		List<List<ESArtifactData>> lists = new ArrayList<>();
		for (int i = 0; i < threadNumber; i++) {
			lists.add(new ArrayList<>());
		}

		List<Future<List<Resource>>> results = new ArrayList<>();
		for (int i = 0; i < list.size(); i++) {
			lists.get(i % threadNumber).add(list.get(i));
		}

		for (int i = 0; i < threadNumber; i++) {

			// Callable<List<Resource>> worker = new
			// MyDesrializabletCallable(lists.get(i), i);
			Callable<List<Resource>> worker = new My3rdPartyDesrializabletCallable(lists.get(i), i);
			Future<List<Resource>> submit = executor.submit(worker);
			results.add(submit);
		}

		long fullSearchStart2 = System.currentTimeMillis();
		for (Future<List<Resource>> future : results) {
			try {
				while (false == future.isDone()) {
					Thread.sleep(1);
				}

			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
		long fullSearchEnd2 = System.currentTimeMillis();
		log.info("esofer time wait to threads finish " + ((fullSearchEnd2 - fullSearchStart2)) + " ms");
		// }
		long fullSearchEnd = System.currentTimeMillis();

		log.info("esofer full desrialize time " + ((fullSearchEnd - fullSearchStart)) + " ms");
		System.out.println("esofer full desrialize time " + ((fullSearchEnd - fullSearchStart)) + " ms");
	}

	public class MyDesrializabletCallable implements Callable<List<Resource>> {

		List<ESArtifactData> list;
		int i;

		public MyDesrializabletCallable(List<ESArtifactData> list, int i) {
			super();
			this.list = list;
			this.i = i;
		}

		@Override
		public List<Resource> call() throws Exception {
			List<Resource> resources = new ArrayList<>();
			long startSer = System.currentTimeMillis();
			long endSer = System.currentTimeMillis();
			long startUnzip = System.currentTimeMillis();
			long endUnzip = System.currentTimeMillis();

			long avgUnzip = 0;
			long avgSer = 0;
			for (ESArtifactData esArtifactData : list) {

				byte[] dataAsArray = esArtifactData.getDataAsArray();
				startUnzip = System.nanoTime();
				dataAsArray = ZipUtil.unzip(dataAsArray);
				endUnzip = System.nanoTime();
				avgUnzip += (endUnzip - startUnzip);

				startSer = System.nanoTime();
				Either<Object, Boolean> deserialize = SerializationUtils.deserialize(dataAsArray);
				endSer = System.nanoTime();
				avgSer += (endSer - startSer);
				// Either<Object, Boolean> deserialize =
				// SerializationUtils.deserialize(esArtifactData.getDataAsArray());
				if (deserialize.isRight()) {
					exit("convertByteArrayToResource " + deserialize.right().value(), 5);
				}

				Resource resource = (Resource) deserialize.left().value();
				resources.add(resource);
				// System.out.println("After desrialize T[" + i + "]resource " +
				// resource.getUniqueId());
			}

			System.out.println("After desrialize average desrialize " + list.size() + " T[" + i + "] "
					+ (avgSer / 1000 / list.size()) + " micro");
			System.out.println(
					"After desrialize average unzip T[" + i + "] " + (avgUnzip / 1000 / list.size()) + " micro");

			////////////////////////
			// maybe register most frequently used classes on conf
			// write
			// byte barray[] = conf.asByteArray(mySerializableObject);
			// read
			// MyObject object = (MyObject)conf.asObject(barray);

			return resources;
		}
	}

	public class My3rdPartyDesrializabletCallable implements Callable<List<Resource>> {

		List<ESArtifactData> list;
		int i;

		public My3rdPartyDesrializabletCallable(List<ESArtifactData> list, int i) {
			super();
			this.list = list;
			this.i = i;
		}

		@Override
		public List<Resource> call() throws Exception {
			List<Resource> resources = new ArrayList<>();
			long startSer = System.currentTimeMillis();
			long endSer = System.currentTimeMillis();
			long startUnzip = System.currentTimeMillis();
			long endUnzip = System.currentTimeMillis();

			long avgUnzip = 0;
			long avgSer = 0;
			for (ESArtifactData esArtifactData : list) {

				byte[] dataAsArray = esArtifactData.getDataAsArray();
				startUnzip = System.nanoTime();
				dataAsArray = ZipUtil.unzip(dataAsArray);
				endUnzip = System.nanoTime();
				avgUnzip += (endUnzip - startUnzip);

				startSer = System.nanoTime();

				Either<Resource, Boolean> deserializeExt = SerializationUtils.deserializeExt(dataAsArray,
						Resource.class, "");

				if (deserializeExt.isLeft()) {
					Resource resource = deserializeExt.left().value();
					// System.out.println("=============================================");
					// System.out.println(resource.getCapabilities().size());
					// System.out.println(resource.getRequirements().size());
					endSer = System.nanoTime();
					avgSer += (endSer - startSer);
					resources.add(resource);
					// System.out.println("After desrialize T[" + i + "]resource
					// " + resource.getUniqueId());
				}
			}

			System.out.println("After desrialize average desrialize " + list.size() + " T[" + i + "] "
					+ (avgSer / 1000 / list.size()) + " micro");
			System.out.println(
					"After desrialize average unzip T[" + i + "] " + (avgUnzip / 1000 / list.size()) + " micro");

			return resources;
		}
	}

}