package com.broadleafdemo.catalog.dataimport;
import org.apache.commons.lang3.StringUtils;
import org.springframework.lang.Nullable;
import org.springframework.util.CollectionUtils;
import com.broadleafcommerce.catalog.domain.CategoryProduct;
import com.broadleafcommerce.catalog.domain.asset.ProductAsset;
import com.broadleafcommerce.catalog.domain.category.Category;
import com.broadleafcommerce.catalog.domain.category.CategoryRef;
import com.broadleafcommerce.catalog.domain.product.Product;
import com.broadleafcommerce.catalog.domain.type.InventoryCheckStrategy;
import com.broadleafcommerce.catalog.domain.type.InventoryType;
import com.broadleafcommerce.catalog.provider.RouteConstants;
import com.broadleafcommerce.catalog.service.CategoryProductService;
import com.broadleafcommerce.catalog.service.CategoryService;
import com.broadleafcommerce.catalog.service.asset.ProductAssetService;
import com.broadleafcommerce.catalog.service.product.ProductService;
import com.broadleafcommerce.common.dataimport.messaging.BatchCompletion;
import com.broadleafcommerce.common.dataimport.messaging.BatchCompletionRecord;
import com.broadleafcommerce.common.dataimport.messaging.BatchCompletionRecordStatus;
import com.broadleafcommerce.common.dataimport.messaging.BatchRecord;
import com.broadleafcommerce.common.dataimport.messaging.BatchRequest;
import com.broadleafcommerce.common.extension.TypeFactory;
import com.broadleafcommerce.common.extension.data.DataRouteByExample;
import com.broadleafcommerce.data.tracking.core.context.ContextInfo;
import com.broadleafcommerce.data.tracking.core.service.BulkPersistenceResponse;
import com.broadleafcommerce.data.tracking.core.service.Update;
import com.broadleafcommerce.data.tracking.core.type.OperationType;
import com.broadleafcommerce.money.util.MonetaryUtils;
import com.broadleafdemo.catalog.service.MyCategoryService;
import com.broadleafdemo.catalog.service.MyProductService;
import java.math.BigDecimal;
import java.security.SecureRandom;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import io.azam.ulidj.ULID;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
@DataRouteByExample(Product.class)
@RequiredArgsConstructor
public class CustomProductBatchHandler implements CustomBatchHandler {
/**
* Expected header fields expected on the CSV Flat File.
*/
public static final String PART_NO = "Part Nbr";
public static final String NAME = "Name";
public static final String SHORT_DESC = "Short Description";
public static final String LONG_DESC = "Long Description";
public static final String MSRP = "SRP";
public static final String IMAGE_URL = "Image URL";
public static final String IS_STOCKING_ONLINE = "Is Stocking Online";
public static final String MODEL_CODE = "Model Code";
public static final String CLASS_CODE = "Class Code";
private static final SecureRandom RANDOM = new SecureRandom();
private final ProductService<Product> productService;
private final ProductAssetService<ProductAsset> productAssetService;
private final CategoryService<Category> categoryService;
private final CategoryProductService<CategoryProduct> categoryProductService;
private final CustomImportContextHelper contextHelper;
private final TypeFactory typeFactory;
@Override
public String getImportType() {
return "MY_CUSTOM_PRODUCT_IMPORT";
}
@Override
public String getDataRouteKey() {
return RouteConstants.Persistence.CATALOG_ROUTE_KEY;
}
@Override
public BatchCompletion handle(BatchRequest batch) {
ContextInfo context = contextHelper.getContext(batch);
ContextInfo readContext = context.withOperationType(context, OperationType.READ);
ContextInfo createContext = context.withOperationType(context, OperationType.CREATE);
ContextInfo updateContext = context.withOperationType(context, OperationType.UPDATE);
// Map of products with Category External ID as the key
Map<String, List<Product>> productMap = new HashMap<>();
Map<String, String> productCorrelationIdMap = new HashMap<>();
Map<String, String> categoryProductCorrelationIdMap = new HashMap<>();
Map<String, String> productAssetCorrelationIdMap = new HashMap<>();
List<ProductAsset> productAssets = new ArrayList<>();
Map<String, Product> existingProductMap =
findExistingProductsByExternalId(batch, readContext);
DiffsDetermined totalDiffs = new DiffsDetermined();
for (BatchRecord record : batch.getRecords()) {
Map<String, String> row = record.getRow();
String partNumber = row.get(PART_NO);
if (existingProductMap.containsKey(partNumber)) {
Product product = existingProductMap.get(partNumber);
totalDiffs.updateAll(determineDiffs(product, row));
productCorrelationIdMap.put(product.getExternalId(), record.getCorrelationId());
continue;
}
Product product = buildProduct(row);
String categoryExternalId = getCategoryExternalId(row);
productMap.computeIfAbsent(categoryExternalId, k -> new ArrayList<>())
.add(product);
productCorrelationIdMap.put(product.getExternalId(), record.getCorrelationId());
categoryProductCorrelationIdMap.put(product.getExternalId(), record.getCorrelationId());
if (StringUtils.isNotBlank(row.get(IMAGE_URL))) {
productAssets.add(buildProductAsset(row, product));
productAssetCorrelationIdMap.put(product.getExternalId(),
record.getCorrelationId());
}
}
BulkPersistenceResponse<Product> createProductResponse =
createProducts(productMap, createContext);
BulkPersistenceResponse<Product> updateProductResponse =
updateProducts(totalDiffs.getUpdateProducts(), updateContext);
BulkPersistenceResponse<CategoryProduct> categoryProductsResponse =
persistCategoryProducts(productMap, createContext, readContext);
BulkPersistenceResponse<ProductAsset> productAssetResponse =
persistProductAssets(productAssets, createContext);
return buildBatchCompletion(batch, createProductResponse,
updateProductResponse,
categoryProductsResponse,
productAssetResponse,
totalDiffs.getNoChangeRecords(),
productCorrelationIdMap,
categoryProductCorrelationIdMap,
productAssetCorrelationIdMap);
}
/**
* Builds a {@link BatchCompletion} object based on the outcomes of the custom import initiated
* by the passed in {@link BatchRequest}
*
* @param batch
* @param createProductResponse
* @param updateProductResponse
* @param categoryProductsResponse
* @param productAssetResponse
* @param noChangeRecords
* @param productCorrelationIdMap
* @param categoryProductCorrelationIdMap
* @param productAssetCorrelationIdMap
* @return
*/
private BatchCompletion buildBatchCompletion(BatchRequest batch,
BulkPersistenceResponse<Product> createProductResponse,
BulkPersistenceResponse<Product> updateProductResponse,
BulkPersistenceResponse<CategoryProduct> categoryProductsResponse,
BulkPersistenceResponse<ProductAsset> productAssetResponse,
Set<String> noChangeRecords,
Map<String, String> productCorrelationIdMap,
Map<String, String> categoryProductCorrelationIdMap,
Map<String, String> productAssetCorrelationIdMap) {
List<BatchCompletionRecord> completions = new ArrayList<>();
completions.addAll(
ImportUtil.getSuccessfulRecords(createProductResponse, productCorrelationIdMap,
Product::getExternalId));
completions.addAll(
ImportUtil.getSuccessfulRecords(updateProductResponse, productCorrelationIdMap,
Product::getExternalId));
completions.addAll(
ImportUtil.getSuccessfulRecords(categoryProductsResponse,
categoryProductCorrelationIdMap,
categoryProduct -> categoryProduct.getProduct().getExternalId()));
completions.addAll(
ImportUtil.getSuccessfulRecords(productAssetResponse,
productAssetCorrelationIdMap,
ProductAsset::getProductId));
completions.addAll(
ImportUtil.getFailedRecords(createProductResponse, productCorrelationIdMap,
Product::getExternalId));
completions.addAll(
ImportUtil.getFailedRecords(updateProductResponse, productCorrelationIdMap,
Product::getExternalId));
completions
.addAll(ImportUtil.getFailedRecords(categoryProductsResponse,
categoryProductCorrelationIdMap,
categoryProduct -> categoryProduct.getProduct().getExternalId()));
completions.addAll(
ImportUtil.getFailedRecords(productAssetResponse, productAssetCorrelationIdMap,
ProductAsset::getProductId));
completions.addAll(getNoChangeRecords(noChangeRecords, productCorrelationIdMap));
completions = combineRecords(completions);
return new BatchCompletion(batch.getId(), batch.getImportId(), completions);
}
/**
* CRUD Helper Methods
*/
private BulkPersistenceResponse<Product> createProducts(Map<String, List<Product>> productMap,
ContextInfo createContext) {
List<Product> products = new ArrayList<>();
productMap.values().forEach(products::addAll);
return productService.createAllAllowingPartialSuccess(products, createContext);
}
/**
* CRUD Helper Methods
*/
private BulkPersistenceResponse<Product> updateProducts(Set<Product> products,
ContextInfo createContext) {
if (products.isEmpty()) {
return new BulkPersistenceResponse<>();
}
Map<String, Product> idMap = products.stream()
.collect(Collectors.toMap(Product::getId, Function.identity(),
(product1, product2) -> product1));
List<Update<Product>> updateProducts = idMap.values().stream()
.map(product -> new Update<>(product.getId(), product))
.collect(Collectors.toList());
return productService.updateAllAllowingPartialSuccess(updateProducts, createContext);
}
/**
* CRUD Helper Methods
*/
private BulkPersistenceResponse<CategoryProduct> persistCategoryProducts(
Map<String, List<Product>> productMap,
ContextInfo createContext,
ContextInfo readContext) {
Map<String, Category> categories = readCategories(productMap.keySet(), readContext);
List<CategoryProduct> categoryProducts = buildCategoryProducts(categories, productMap);
return categoryProductService.createAllAllowingPartialSuccess(categoryProducts,
createContext);
}
/**
* CRUD Helper Methods
*/
private BulkPersistenceResponse<ProductAsset> persistProductAssets(
List<ProductAsset> productAssets,
ContextInfo createContext) {
return productAssetService.createAllAllowingPartialSuccess(productAssets,
createContext);
}
/**
* Returns a map with the product external ID as the key and the product as the value.
*
* @param batch
* @param readContext
* @return
*/
private Map<String, Product> findExistingProductsByExternalId(BatchRequest batch,
ContextInfo readContext) {
List<String> externalIds = batch.getRecords().stream()
.map(record -> record.getRow().get(PART_NO))
.collect(Collectors.toList());
Map<String, Product> productMap = new HashMap<>();
List<Product> products = ((MyProductService<Product>) productService)
.readAllByExternalIds(externalIds, readContext);
for (Product product : products) {
productMap.put(product.getExternalId(), product);
}
return productMap;
}
/**
* Returns a map with the category external ID as the key and the category as the value.
*
* @param categoryExternalIds A set of category external IDs.
* @param contextInfo
* @return A map of category external IDs.
*/
private Map<String, Category> readCategories(Set<String> categoryExternalIds,
ContextInfo contextInfo) {
List<Category> categories =
((MyCategoryService<Category>) categoryService)
.readAllByExternalIds(categoryExternalIds, contextInfo);
Map<String, Category> categoryMap = new HashMap<>();
if (CollectionUtils.isEmpty(categories)) {
return categoryMap;
}
for (Category category : categories) {
categoryMap.put(category.getExternalId(), category);
}
return categoryMap;
}
/**
* Builds the Category Product Relationship Assumes that the Category already exists in the
* system.
*
* @param categoriesMap
* @param productMap
* @return
*/
private List<CategoryProduct> buildCategoryProducts(Map<String, Category> categoriesMap,
Map<String, List<Product>> productMap) {
if (productMap.isEmpty()) {
return Collections.emptyList();
}
List<CategoryProduct> categoryProducts = new ArrayList<>();
for (String categoryExternalId : categoriesMap.keySet()) {
Category category = categoriesMap.get(categoryExternalId);
for (Product product : productMap.get(categoryExternalId)) {
CategoryProduct categoryProduct = typeFactory.get(CategoryProduct.class);
categoryProduct.setProduct(product);
categoryProduct
.setCategory(typeFactory.get(CategoryRef.class).fromCategory(category));
categoryProduct.setId(ULID.random(RANDOM));
categoryProducts.add(categoryProduct);
}
}
return categoryProducts;
}
/**
* For this example, we'll assume a simple 2-level category: "Model" and "Class" The categories
* are determined by a property called "Model" (parent category) and a second-level category
* called "Class" (child category).
*
* These will be stored in Category#getExternalId as ${model}-{$class} (model dash class)
*
* @param row
* @return
*/
private String getCategoryExternalId(Map<String, String> row) {
String model = row.get(MODEL_CODE);
String classCode = row.get(CLASS_CODE);
if (classCode == null) {
return model;
}
return model + "-" + classCode;
}
/**
* Sample builder method to convert a CSV row into a {@link Product}
*
* @param row
* @return
*/
private Product buildProduct(Map<String, String> row) {
Product product = new Product();
product.setId(ULID.random(RANDOM));
product.setExternalId(row.get(PART_NO));
product.setName(row.get(NAME));
product.setSku(row.get(PART_NO));
product.setDescription(getDescription(row));
product.setMsrp(MonetaryUtils.toAmount(getBigDecimal(row, MSRP),
MonetaryUtils.defaultCurrency().getCurrencyCode()));
product.setDefaultPrice(
MonetaryUtils.toAmount(getBigDecimal(row, MSRP),
MonetaryUtils.defaultCurrency().getCurrencyCode()));
product.setActiveStartDate(Instant.now());
product.setDiscountable(true);
product.setIndividuallySold(true);
product.setMetaTitle(row.get(SHORT_DESC));
product.setMetaDescription(row.get(LONG_DESC));
product.setOnline(true);
product.setSearchable(true);
product.setUri(ImportUtil.getUrlStr(row.get(PART_NO)));
product.setInventoryType(InventoryType.PHYSICAL.name());
product.setCurrency(MonetaryUtils.defaultCurrency());
product.setAvailableOnline(getBoolean(row, IS_STOCKING_ONLINE));
product.setInventoryCheckStrategy(InventoryCheckStrategy.NEVER.name());
return product;
}
/**
* Sample builder method to convert a CSV row into a {@link ProductAsset}
*
* @param row
* @return
*/
private ProductAsset buildProductAsset(Map<String, String> row, Product product) {
ProductAsset productAsset = typeFactory.get(ProductAsset.class);
productAsset.setId(row.get(PART_NO));
productAsset.setAltText(row.get(PART_NO));
productAsset.setPrimary(true);
productAsset.setProductId(product.getId());
productAsset.setProvider("UNSPECIFIED");
productAsset.setTitle(row.get(PART_NO));
productAsset.setType("IMAGE");
productAsset.setUrl(row.get(IMAGE_URL));
return productAsset;
}
/**
* Row processing helper method
*/
@Nullable
private BigDecimal getBigDecimal(Map<String, String> row, String key) {
String value = row.get(key);
return value == null ? null : new BigDecimal(value);
}
/**
* Row processing helper method
*/
private String getDescription(Map<String, String> row) {
String shortDesc =
org.springframework.util.StringUtils.trimTrailingWhitespace(row.get(SHORT_DESC));
String longDesc =
org.springframework.util.StringUtils.trimTrailingWhitespace(row.get(LONG_DESC));
if (StringUtils.isEmpty(longDesc)) {
return shortDesc;
}
return shortDesc + " " + longDesc;
}
/**
* Row processing helper method
*/
private boolean getBoolean(Map<String, String> row, String key) {
String val = row.get(key);
return "Y".equals(val) || "Yes".equals(val) || "true".equalsIgnoreCase(val);
}
/**
* Simple example to granularly determine row variation against an existing product
*
* @param existing
* @param row
* @return
*/
private DiffsDetermined determineDiffs(Product existing, Map<String, String> row) {
DiffsDetermined diffsDetermined = new DiffsDetermined();
boolean diffExists = false;
Product potentialUpdates = buildProduct(row);
if (!StringUtils.equals(existing.getDescription(), potentialUpdates.getDescription())) {
existing.setDescription(potentialUpdates.getDescription());
diffExists = true;
}
if (!Objects.equals(existing.getMsrp(), potentialUpdates.getMsrp())) {
existing.setMsrp(potentialUpdates.getMsrp());
diffExists = true;
}
if (!Objects.equals(existing.getDefaultPrice(), potentialUpdates.getDefaultPrice())) {
existing.setDefaultPrice(potentialUpdates.getDefaultPrice());
diffExists = true;
}
if (!StringUtils.equals(existing.getMetaTitle(), potentialUpdates.getMetaTitle())) {
existing.setMetaTitle(potentialUpdates.getMetaTitle());
diffExists = true;
}
if (!StringUtils.equals(existing.getMetaDescription(),
potentialUpdates.getMetaDescription())) {
existing.setMetaDescription(potentialUpdates.getMetaDescription());
diffExists = true;
}
if (existing.isAvailableOnline() != potentialUpdates.isAvailableOnline()) {
existing.setAvailableOnline(potentialUpdates.isAvailableOnline());
diffExists = true;
}
if (!StringUtils.equals(existing.getInventoryCheckStrategy(),
potentialUpdates.getInventoryCheckStrategy())) {
existing.setInventoryCheckStrategy(potentialUpdates.getInventoryCheckStrategy());
diffExists = true;
}
if (diffExists) {
diffsDetermined.getUpdateProducts()
.add(existing);
} else {
diffsDetermined.getNoChangeRecords()
.add(existing.getExternalId());
}
return diffsDetermined;
}
private Collection<? extends BatchCompletionRecord> getNoChangeRecords(
Set<String> noChangeProducts,
Map<String, String> productCorrelationIdMap) {
return noChangeProducts.stream()
.map(noChange -> new BatchCompletionRecord(
productCorrelationIdMap.get(noChange),
BatchCompletionRecordStatus.SUCCESS))
.collect(Collectors.toSet());
}
@Data
@NoArgsConstructor
@AllArgsConstructor
private class DiffsDetermined {
Set<Product> updateProducts = new HashSet<>();
Set<String> noChangeRecords = new HashSet<>();
boolean missingRecords = false;
String missingRecordError;
public void updateAll(DiffsDetermined otherDiffs) {
this.getUpdateProducts()
.addAll(otherDiffs.getUpdateProducts());
this.getNoChangeRecords()
.addAll(otherDiffs.getNoChangeRecords());
}
}
}