Skip to content

Commit

Permalink
Merge pull request #791 from motech-implementations/obd_file_bifurcation
Browse files Browse the repository at this point in the history 
Bifurcation of obd files to separate priority subscribers to enable b…
  • Loading branch information
@gudipatiharitha
gudipatiharitha authored Jul 9, 2019
2 parents 87a81d4 + be31ab7 commit 505ef19
Show file tree
Hide file tree
Showing 8 changed files with 331 additions and 169 deletions.
8 changes: 5 additions & 3 deletions imi/src/main/java/org/motechproject/nms/imi/service/TargetFileService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import org.motechproject.nms.imi.web.contract.FileProcessedStatusRequest;
import org.motechproject.nms.kilkari.domain.SubscriptionOrigin;

import java.util.HashMap;

/**
* Creating the targetFile: a csv file containing all the phone numbers to be called by the IVR system
*/
Expand All @@ -12,9 +14,7 @@ public interface TargetFileService {
* Probably only to be called by an IT. This service's constructor sets the repeating
* nms.obd.generate_target_file MOTECH event which triggers the daily generation of the targetFile.
*/
TargetFileNotification generateTargetFile();


HashMap<String, TargetFileNotification> generateTargetFile(boolean split);
/**
* The IVR system invoked the obdFileProcessedStatusNotification http endpoint signalling the completion of the
* processing of the targetFile we generated
Expand All @@ -32,4 +32,6 @@ public interface TargetFileService {
* @return the IMI ServiceID
*/
String serviceIdFromOrigin(boolean freshCall, SubscriptionOrigin origin);

String serviceIdFromOriginJh(boolean freshCall, SubscriptionOrigin origin);
}
363 changes: 239 additions & 124 deletions imi/src/main/java/org/motechproject/nms/imi/service/impl/TargetFileServiceImpl.java
View file

Large diffs are not rendered by default.

19 changes: 12 additions & 7 deletions imi/src/main/java/org/motechproject/nms/imi/web/ImiController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.springframework.web.bind.annotation.ResponseStatus;

import javax.servlet.http.HttpServletRequest;
import java.util.HashMap;
import java.util.regex.Pattern;

/**
Expand All @@ -48,6 +49,7 @@ public class ImiController {
public static final String INVALID_STATUS_ENUM = "Can not construct instance of " +
"org.motechproject.nms.imi.domain.FileProcessedStatus from String value";
public static final Pattern TARGET_FILENAME_PATTERN = Pattern.compile("OBD_NMS_[0-9]{14}\\.csv");
public static final Pattern TARGET_FILENAME_PATTERN_JH = Pattern.compile("OBD_NMS_[0-9]{14}JH\\.csv");
public static final String IVR_INTERACTION_LOG = "IVR INTERACTION: %s";

private static final Logger LOGGER = LoggerFactory.getLogger(ImiController.class);
Expand All @@ -59,6 +61,8 @@ public class ImiController {
private FileAuditRecordDataService fileAuditRecordDataService;
private AlertService alertService;

public static final String generateJhFile = "imi.obd_bifurcate";


@Autowired
public ImiController(SettingsFacade settingsFacade, CdrFileService cdrFileService, AlertService alertService,
Expand Down Expand Up @@ -88,7 +92,8 @@ private static boolean validateFieldPresent(StringBuilder errors, String fieldNa
// verify the passed targetFileName is valid
private static boolean validateTargetFileName(StringBuilder errors, String targetFileName) {
if (validateFieldPresent(errors, "fileName", targetFileName)) {
if (TARGET_FILENAME_PATTERN.matcher(targetFileName).matches()) {
if (TARGET_FILENAME_PATTERN.matcher(targetFileName).matches() ||
TARGET_FILENAME_PATTERN_JH.matcher(targetFileName).matches() ) {
return true;
} else {
errors.append(String.format(INVALID, "fileName"));
Expand Down Expand Up @@ -148,13 +153,13 @@ public String generateTargetFile() {

LOGGER.debug("/generateTargetFile (GET)");
try {
TargetFileNotification tfn = targetFileService.generateTargetFile();
LOGGER.debug("targetFileService.generateTargetFile() done");
HashMap<String, TargetFileNotification> tfn = targetFileService.generateTargetFile(Boolean.parseBoolean(settingsFacade.getProperty(generateJhFile)));
LOGGER.debug("targetFileService.generateTargetFile() done");

return tfn == null ? "null" : tfn.getFileName();
} catch(Exception e) {
LOGGER.error(e.getMessage(),e);
throw e;
return tfn == null ? "null" : tfn.values().toString();
}catch(Exception e){
LOGGER.error(e.getMessage(), e);
throw e;
}

}
Expand Down
14 changes: 11 additions & 3 deletions imi/src/main/resources/imi.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ imi.target_file_time=05:00
imi.target_file_sec_interval=86400

#local directory where target files should be generated
imi.local_obd_dir=obd-files-local
imi.local_obd_dir=/usr/local/imi/obd

#remote directory where target files should be copied to
imi.remote_obd_dir=obd-files-remote
imi.remote_obd_dir=/usr/local/imi/obd-remote

#local directory where CDR files should be copied to
imi.local_cdr_dir=cdr-files-local
Expand Down Expand Up @@ -44,6 +44,11 @@ imi.fresh_no_check_dnd=Retryonroute1noDND_1444132196827
imi.retry_check_dnd=Retryonroute2noDND_1444132213332
imi.retry_no_check_dnd= Retryonroute2noDND_1444132213332

imi.fresh_check_dnd_jh= jhrRetryonroute1noDND_1560947525746
imi.fresh_no_check_dnd_jh=jhrRetryonroute1noDND_1560947525746
imi.retry_check_dnd_jh=jhrRetryonroute2noDND_1560947563925
imi.retry_no_check_dnd_jh= jhrRetryonroute2noDND_1560947563925

#maximum number of errors allowed in a CDR file after which all errors are ignored so as to not overwhelm the
#audit table & the tomcat log file
imi.max_cdr_error_count=100
Expand Down Expand Up @@ -85,4 +90,7 @@ imi.distributed_csr_processing=false

# CSR chunk size - any number {n} greater than one will enable chunk processing, where sets (or chunks) of {n} CSRs are
# distributed for processing, regardless of the value of imi.distributed_csr_processing
imi.csr_chunk_size=1000
imi.csr_chunk_size=1000

#set to false if bifurcation of obd files is not required
imi.obd_bifurcate=true
1 change: 1 addition & 0 deletions kilkari/src/main/java/org/motechproject/nms/kilkari/service/SubscriptionService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ Subscription createSubscription(Subscriber subscriber, long callingNumber, Langu
*/
List<Subscription> findActiveSubscriptionsForDay(DayOfTheWeek dayOfTheWeek, long offset, int rowCount);

List<String> findJhSubscriptionIds();

/**
* Get the list of pending subscriptions that starts after the specified date.
Expand Down
25 changes: 25 additions & 0 deletions ...ari/src/main/java/org/motechproject/nms/kilkari/service/impl/SubscriptionServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1011,6 +1011,31 @@ public List<Subscription> execute(Query query) {
return subscriptions;
}

@Override
public List<String> findJhSubscriptionIds() {
SqlQueryExecution<List<String>> queryExecution = new SqlQueryExecution<List<String>>() {
@Override
public String getSqlQuery() {
String query = "select subscriptionId from (select subscriptionId from nms_mcts_mothers a LEFT JOIN " +
"nms_subscribers b on b.mother_id_oid=a.id LEFT JOIN nms_subscriptions c on c.subscriber_id_oid = b.id where rchId like 'JH%' " +
"and subscriptionPack_id_OID = 1 UNION ALL select subscriptionId from nms_mcts_children a LEFT JOIN " +
"nms_subscribers b on b.child_id_oid=a.id LEFT JOIN nms_subscriptions c on c.subscriber_id_oid = b.id " +
"where rchId like 'JH%' and subscriptionPack_id_OID = 2) as a;";
LOGGER.debug(KilkariConstants.SQL_QUERY_LOG, query);
return query;
}

@Override
public List<String> execute(Query query) {
List<String> ids = (List<String>) query.execute();
return ids;
}
};
List<String> subscriptionIds = subscriptionDataService.executeSQLQuery(queryExecution);
return subscriptionIds;
}



public List<Subscription> findPendingSubscriptionsFromDate(DateTime startDate, int page, int pageSize) {
return subscriptionDataService.findByStatusAndStartDate(SubscriptionStatus.PENDING_ACTIVATION, startDate, new QueryParams(page, pageSize));
Expand Down
9 changes: 6 additions & 3 deletions testing/src/test/java/org/motechproject/nms/testing/it/imi/ImiControllerCdrBundleIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
Expand Down Expand Up @@ -100,6 +101,8 @@ public class ImiControllerCdrBundleIT extends BasePaxIT {
private String localObdDirBackup;
private String remoteObdDirBackup;
private CdrHelper helper;
private static final String Jh = "JH";
private static final String non_Jh = "NON-JH";


@Before
Expand Down Expand Up @@ -511,12 +514,12 @@ public void verifyTargetFileNameRoundTrip() throws IOException, InterruptedExcep
SubscriptionOrigin.MCTS_IMPORT);
transactionManager.commit(status);

TargetFileNotification tfn = targetFileService.generateTargetFile();
assertNotNull(tfn);
HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);
assertNotNull(tfn.get(non_Jh));

CdrHelper cdrHelper = new CdrHelper(settingsService, subscriberService,subscriptionService, subscriberDataService,
subscriptionPackDataService, languageDataService, languageService, circleDataService, stateDataService,
districtDataService, fileAuditRecordDataService, districtService, tfn.getFileName());
districtDataService, fileAuditRecordDataService, districtService, tfn.get(non_Jh).getFileName());

helper.makeCdrs(1, 0, 0, 0);
File remoteCdrFile = helper.makeRemoteCdrFile();
Expand Down
61 changes: 32 additions & 29 deletions testing/src/test/java/org/motechproject/nms/testing/it/imi/TargetFileServiceBundleIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import java.nio.file.Files;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import static org.junit.Assert.assertEquals;
Expand All @@ -70,6 +71,8 @@ public class TargetFileServiceBundleIT extends BasePaxIT {

private static final String LOCAL_OBD_DIR = "imi.local_obd_dir";
private static final String REMOTE_OBD_DIR = "imi.remote_obd_dir";
private static final String Jh = "JH";
private static final String non_Jh = "NON-JH";


private String localObdDirBackup;
Expand Down Expand Up @@ -175,9 +178,9 @@ public void createLargeFile() {
));
}

TargetFileNotification tfn = targetFileService.generateTargetFile();
assertNotNull(tfn);
getLogger().debug("Generated {}", tfn.getFileName());
HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);
assertNotNull(tfn.get(non_Jh));
getLogger().debug("Generated {}", tfn.values().toString());
}

// To check that target file should contain correct weekID according to LMP of the subscriber.
Expand All @@ -198,10 +201,10 @@ public void verifyFT151() throws NoSuchAlgorithmException, IOException {
List<String> contents = new ArrayList<>();
String line;

TargetFileNotification tfn = targetFileService.generateTargetFile();
HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);

File targetDir = new File(settingsService.getSettingsFacade().getProperty("imi.local_obd_dir"));
File targetFile = new File(targetDir, tfn.getFileName());
File targetFile = new File(targetDir, tfn.get(non_Jh).getFileName());
int recordCount = 0;
boolean header = true;
try (InputStream is = Files.newInputStream(targetFile.toPath());
Expand All @@ -218,8 +221,8 @@ public void verifyFT151() throws NoSuchAlgorithmException, IOException {

String checksum = ChecksumHelper.checksum(targetFile);

assertEquals((int)tfn.getRecordsCount(), recordCount);
assertEquals(tfn.getChecksum(), checksum);
assertEquals((int)tfn.get(non_Jh).getRecordsCount(), recordCount);
assertEquals(tfn.get(non_Jh).getChecksum(), checksum);
assertTrue("w6_1".equals(contents.get(0)));
assertEquals(1, recordCount);
}
Expand All @@ -242,10 +245,10 @@ public void verifyFT152() throws NoSuchAlgorithmException, IOException {
List<String> contents = new ArrayList<>();
String line;

TargetFileNotification tfn = targetFileService.generateTargetFile();
HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);

File targetDir = new File(settingsService.getSettingsFacade().getProperty("imi.local_obd_dir"));
File targetFile = new File(targetDir, tfn.getFileName());
File targetFile = new File(targetDir, tfn.get(non_Jh).getFileName());
int recordCount = 0;
boolean header = true;
try (InputStream is = Files.newInputStream(targetFile.toPath());
Expand All @@ -261,8 +264,8 @@ public void verifyFT152() throws NoSuchAlgorithmException, IOException {
}

String checksum = ChecksumHelper.checksum(targetFile);
assertEquals((int)tfn.getRecordsCount(), recordCount);
assertEquals(tfn.getChecksum(), checksum);
assertEquals((int)tfn.get(non_Jh).getRecordsCount(), recordCount);
assertEquals(tfn.get(non_Jh).getChecksum(), checksum);
assertTrue("w5_1".equals(contents.get(0)));

//update the date of birth of the subscriber
Expand All @@ -272,9 +275,9 @@ public void verifyFT152() throws NoSuchAlgorithmException, IOException {
subscriberService.updateStartDate(subscriber2);

// again generate the target file to check correct weekId is picked after DOB is changed.
tfn = targetFileService.generateTargetFile();
tfn = targetFileService.generateTargetFile(false);
contents.clear();
targetFile = new File(targetDir, tfn.getFileName());
targetFile = new File(targetDir, tfn.get(non_Jh).getFileName());
recordCount = 0;
header = true;
try (InputStream is = Files.newInputStream(targetFile.toPath());
Expand All @@ -289,8 +292,8 @@ public void verifyFT152() throws NoSuchAlgorithmException, IOException {
}
}
checksum = ChecksumHelper.checksum(targetFile);
assertEquals((int)tfn.getRecordsCount(), recordCount);
assertEquals(tfn.getChecksum(), checksum);
assertEquals((int)tfn.get(non_Jh).getRecordsCount(), recordCount);
assertEquals(tfn.get(non_Jh).getChecksum(), checksum);
assertTrue("w4_1".equals(contents.get(0)));
assertEquals(1, recordCount);
}
Expand All @@ -313,10 +316,10 @@ public void verifyFT190() throws NoSuchAlgorithmException, IOException {
List<String> contents = new ArrayList<>();
String line;

TargetFileNotification tfn = targetFileService.generateTargetFile();
HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);

File targetDir = new File(settingsService.getSettingsFacade().getProperty("imi.local_obd_dir"));
File targetFile = new File(targetDir, tfn.getFileName());
File targetFile = new File(targetDir, tfn.get(non_Jh).getFileName());
int recordCount = 0;
boolean header = true;
try (InputStream is = Files.newInputStream(targetFile.toPath());
Expand All @@ -332,8 +335,8 @@ public void verifyFT190() throws NoSuchAlgorithmException, IOException {
}

String checksum = ChecksumHelper.checksum(targetFile);
assertEquals((int)tfn.getRecordsCount(), recordCount);
assertEquals(tfn.getChecksum(), checksum);
assertEquals((int)tfn.get(non_Jh).getRecordsCount(), recordCount);
assertEquals(tfn.get(non_Jh).getChecksum(), checksum);
assertTrue("w1_1.wav".equals(contents.get(0)));
}

Expand All @@ -355,10 +358,10 @@ public void verifyFT191() throws NoSuchAlgorithmException, IOException {
List<String> contents = new ArrayList<>();
String line;

TargetFileNotification tfn = targetFileService.generateTargetFile();
HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);

File targetDir = new File(settingsService.getSettingsFacade().getProperty("imi.local_obd_dir"));
File targetFile = new File(targetDir, tfn.getFileName());
File targetFile = new File(targetDir, tfn.get(non_Jh).getFileName());
int recordCount = 0;
boolean header = true;
try (InputStream is = Files.newInputStream(targetFile.toPath());
Expand All @@ -374,8 +377,8 @@ public void verifyFT191() throws NoSuchAlgorithmException, IOException {
}

String checksum = ChecksumHelper.checksum(targetFile);
assertEquals((int)tfn.getRecordsCount(), recordCount);
assertEquals(tfn.getChecksum(), checksum);
assertEquals((int)tfn.get(non_Jh).getRecordsCount(), recordCount);
assertEquals(tfn.get(non_Jh).getChecksum(), checksum);
assertTrue("w1_1.wav".equals(contents.get(0)));

}
Expand All @@ -395,9 +398,9 @@ public void testChecksumsVaryWithFileContent() throws NoSuchAlgorithmException,

transactionManager.commit(status);

TargetFileNotification tfn1 = targetFileService.generateTargetFile();
assertNotNull(tfn1);
getLogger().debug(tfn1.toString());
HashMap<String,TargetFileNotification> tfn1 = targetFileService.generateTargetFile(false);
assertNotNull(tfn1.get(non_Jh));
getLogger().debug(tfn1.get(non_Jh).toString());

// Sleep two seconds so the file names are different
Thread.sleep(2000L);
Expand All @@ -413,11 +416,11 @@ public void testChecksumsVaryWithFileContent() throws NoSuchAlgorithmException,

transactionManager.commit(status);

TargetFileNotification tfn2 = targetFileService.generateTargetFile();
assertNotNull(tfn2);
HashMap<String,TargetFileNotification> tfn2 = targetFileService.generateTargetFile(false);
assertNotNull(tfn2.get(non_Jh));
getLogger().debug(tfn2.toString());

assertFalse(tfn1.getChecksum().equals(tfn2.getChecksum()));
assertFalse(tfn1.get(non_Jh).getChecksum().equals(tfn2.get(non_Jh).getChecksum()));
}

}

0 comments on commit 505ef19

Please sign in to comment.