Skip to content

Commit

Permalink
Major updates tiemout + impersonation
Browse files Browse the repository at this point in the history 
Upgraded credential
Added Timeout
Added impersonation
  • Loading branch information
@jfmonteil
jfmonteil committed Jan 1, 2022
1 parent 9c32212 commit c9282cf
Show file tree
Hide file tree
Showing 11 changed files with 389 additions and 183 deletions.
Binary file added delivery/PentahoGoogleSheetsPlugin-5-distribution.zip
View file
Binary file not shown.
18 changes: 13 additions & 5 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.pentaho</groupId>
<artifactId>PentahoGoogleSheetsPlugin</artifactId>
<version>4.2</version>
<version>5</version>
<packaging>jar</packaging>
<name>"Pentaho Google Sheets"</name>
<description>"Google Sheets Pentaho Plugin API V4"</description>
Expand Down Expand Up @@ -50,17 +50,15 @@
<version>${pentaho.kettle.version}</version>
<scope>provided</scope>
</dependency>


<dependency>
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client</artifactId>
<version>1.23.0</version>
<version>1.32.2</version>
</dependency>
<dependency>
<groupId>com.google.oauth-client</groupId>
<artifactId>google-oauth-client-jetty</artifactId>
<version>1.23.0</version>
<version>1.32.1</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
Expand All @@ -72,6 +70,16 @@
<artifactId>google-api-services-drive</artifactId>
<version>v3-rev20191108-1.30.3</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-sqladmin</artifactId>
<version>v1-rev20210816-1.32.1</version>
</dependency>
<dependency>
<groupId>com.google.auth</groupId>
<artifactId>google-auth-library-oauth2-http</artifactId>
<version>1.3.0</version>
</dependency>
</dependencies>
<build>
<resources>
Expand Down
18 changes: 12 additions & 6 deletions samples/inpout-output.ktr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -456,7 +456,10 @@
</partitioning>
<worksheetId>Sheet1</worksheetId>
<spreadsheetKey>1mDcSKwt7MnPv5sEdB5GBV4L2t06JdXGSAnxbdaO1rWE</spreadsheetKey>
<jsonCredentialPath>C:\Users\jmonteil\.kettle/client_secret.json</jsonCredentialPath>
<jsonCredentialPath>C:\googlekeys\pentaho-sheets-261911-abd85b3badca.json</jsonCredentialPath>
<timeout>5</timeout>
<impersonation/>
<appName>pentaho-sheets</appName>
<sampleFields>100</sampleFields>
<fields>
<field>
Expand Down Expand Up @@ -492,7 +495,7 @@
<group>,</group>
<position>-1</position>
<length>-1</length>
<precision>0</precision>
<precision>-1</precision>
<trim_type>none</trim_type>
</field>
<field>
Expand All @@ -516,7 +519,7 @@
<group>,</group>
<position>-1</position>
<length>-1</length>
<precision>0</precision>
<precision>-1</precision>
<trim_type>none</trim_type>
</field>
<field>
Expand All @@ -540,7 +543,7 @@
<group>,</group>
<position>-1</position>
<length>-1</length>
<precision>0</precision>
<precision>-1</precision>
<trim_type>none</trim_type>
</field>
<field>
Expand All @@ -552,7 +555,7 @@
<group>,</group>
<position>-1</position>
<length>-1</length>
<precision>0</precision>
<precision>-1</precision>
<trim_type>none</trim_type>
</field>
<field>
Expand Down Expand Up @@ -629,13 +632,16 @@
<method>none</method>
<schema_name/>
</partitioning>
<jsonCredentialPath>C:\Users\jmonteil\.kettle/client_secret.json</jsonCredentialPath>
<jsonCredentialPath>C:\googlekeys\pentaho-sheets-261911-abd85b3badca.json</jsonCredentialPath>
<worksheetId>fonk</worksheetId>
<spreadsheetKey>1mDcSKwt7MnPv5sEdB5GBV4L2t06JdXGSAnxbdaO1rWE</spreadsheetKey>
<CREATE>false</CREATE>
<APPEND>false</APPEND>
<SHAREEMAIL/>
<SHAREDOMAIN/>
<timeout>5</timeout>
<impersonation/>
<appName>pentaho-sheets</appName>
<attributes/>
<cluster_schema/>
<remotesteps>
Expand Down
Binary file modified screenshots/PentahoGoogleSheetInput-Credential.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
75 changes: 30 additions & 45 deletions .../org/pentaho/di/trans/steps/pentahogooglesheets/PentahoGoogleSheetsPluginCredentials.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,75 +19,60 @@
package org.pentaho.di.trans.steps.pentahogooglesheets;

import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.core.exception.KettleFileException;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.*;
import org.pentaho.di.core.Const;



import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp;
import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver;
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow;
import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets;
import com.google.auth.http.HttpCredentialsAdapter;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.util.store.FileDataStoreFactory;
import com.google.api.services.sheets.v4.Sheets;
import com.google.api.services.sheets.v4.SheetsScopes;
import com.google.api.services.sheets.v4.model.ValueRange;
import com.google.api.client.util.Base64;

import com.google.auth.oauth2.GoogleCredentials;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.services.sqladmin.*;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.*;
import com.google.api.client.http.*;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.security.GeneralSecurityException;
import java.util.Collections;
import java.util.List;
import java.util.ArrayList;
import java.util.Arrays;
import java.security.KeyStore;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;


/**
* Describe your step plugin.
*
*/
public class PentahoGoogleSheetsPluginCredentials {

public class PentahoGoogleSheetsPluginCredentials {



public static Credential getCredentialsJson(String scope,String jsonCredentialPath) throws IOException {

Credential credential=null;
//InputStream in = PentahoGoogleSheetsPluginCredentials.class.getResourceAsStream("/plugins/pentaho-googledrive-vfs/credentials/client_secret.json");//pentaho-sheets-261911-18ce0057e3d3.json
//logBasic("Getting credential json file from :"+Const.getKettleDirectory());
public static HttpCredentialsAdapter getCredentialsJson(String scope,String jsonCredentialPath,String impersonation) throws IOException {

GoogleCredentials credential=null;

InputStream in=null;
try{
in = KettleVFS.getInputStream(jsonCredentialPath);//Const.getKettleDirectory() + "/client_secret.json");
} catch (Exception e) {
//throw new KettleFileException("Exception",e.getMessage(),e);
}

if (in == null) {
throw new FileNotFoundException("Resource not found:"+ jsonCredentialPath);
}
credential = GoogleCredential.fromStream(in).createScoped(Collections.singleton(scope));
return credential;
if(impersonation.isEmpty()) {
credential = GoogleCredentials.fromStream(in).createScoped(Collections.singleton(scope));
}
else credential = GoogleCredentials.fromStream(in)
.createScoped(Collections.singleton(SQLAdminScopes.SQLSERVICE_ADMIN)).createDelegated(impersonation);

return new HttpCredentialsAdapter(credential);
}

public static HttpRequestInitializer setHttpTimeout(final HttpRequestInitializer requestInitializer, final String timeout) {
return new HttpRequestInitializer() {
@Override
public void initialize(HttpRequest httpRequest) throws IOException {
Integer TO=Integer.parseInt(timeout);
requestInitializer.initialize(httpRequest);
httpRequest.setConnectTimeout(TO * 60000); // 10 minutes connect timeout
httpRequest.setReadTimeout(TO * 60000); // 10 minutes read timeout
}
};
}


Expand Down
21 changes: 8 additions & 13 deletions ...n/java/org/pentaho/di/trans/steps/pentahogooglesheets/PentahoGoogleSheetsPluginInput.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@



import com.google.api.client.http.HttpRequestInitializer;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMeta;
Expand All @@ -40,7 +41,7 @@
import com.google.api.client.http.javanet.NetHttpTransport;

import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.json.gson.GsonFactory;
import com.google.api.services.sheets.v4.Sheets;
import com.google.api.services.sheets.v4.model.ValueRange;
import com.google.api.services.sheets.v4.SheetsScopes;
Expand Down Expand Up @@ -76,15 +77,7 @@ public class PentahoGoogleSheetsPluginInput extends BaseStep implements StepInte
public PentahoGoogleSheetsPluginInput( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans ) {
super( stepMeta, stepDataInterface, copyNr, transMeta, trans );
}

/**
* Initialize and do work where other steps need to wait for...
*
* @param stepMetaInterface
* The metadata to work with
* @param stepDataInterface
* The data to initialize
*/

@Override
public boolean init( StepMetaInterface smi, StepDataInterface sdi ) {

Expand All @@ -97,16 +90,18 @@ public boolean init( StepMetaInterface smi, StepDataInterface sdi ) {
String scope=SheetsScopes.SPREADSHEETS_READONLY;

try {
JSON_FACTORY = JacksonFactory.getDefaultInstance();
JSON_FACTORY = GsonFactory.getDefaultInstance();
HTTP_TRANSPORT=GoogleNetHttpTransport.newTrustedTransport();
} catch (Exception e) {
logError("Exception",e.getMessage(),e);
}


if (super.init(smi, sdi)) {
try {
Sheets service = new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, PentahoGoogleSheetsPluginCredentials.getCredentialsJson(scope,environmentSubstitute(meta.getJsonCredentialPath()))).setApplicationName(APPLICATION_NAME).build();
try {
HttpRequestInitializer credential=PentahoGoogleSheetsPluginCredentials.getCredentialsJson(scope,environmentSubstitute(meta.getJsonCredentialPath()),environmentSubstitute(meta.getImpersonation()));

Sheets service = new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, PentahoGoogleSheetsPluginCredentials.setHttpTimeout(credential,environmentSubstitute(meta.getTimeout()))).setApplicationName(APPLICATION_NAME).build();
String range=environmentSubstitute(meta.getWorksheetId());
ValueRange response = service.spreadsheets().values().get(environmentSubstitute(meta.getSpreadsheetKey()),range).execute();
if(response==null) {
Expand Down
56 changes: 51 additions & 5 deletions ...va/org/pentaho/di/trans/steps/pentahogooglesheets/PentahoGoogleSheetsPluginInputMeta.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,18 @@ public PentahoGoogleSheetsPluginInputMeta() {

@Injection( name = "worksheetId", group = "SHEET" )
private String worksheetId;


@Injection( name = "timeout", group = "SHEET" )
private String timeout;

@Injection( name = "impersonation", group = "SHEET" )
private String impersonation;

@Injection( name = "sampleFields", group = "INPUT_Fields" )
private Integer sampleFields;

@Injection( name = "appName", group = "SHEET" )
private String appName;

@InjectionDeep
private PentahoGoogleSheetsPluginInputFields[] inputFields;
Expand All @@ -103,6 +112,8 @@ public void setDefault() {
this.spreadsheetKey = "";
this.worksheetId = "";
this.jsonCredentialPath = Const.getKettleDirectory()+ "/client_secret.json";
this.timeout="5";
this.impersonation="";
this.sampleFields=100;

}
Expand Down Expand Up @@ -142,11 +153,31 @@ public int getSampleFields() {
public void setSampleFields(int sampleFields) {
this.sampleFields = sampleFields;
}

public void allocate(int nrfields) {

public void setTimeout(String timeout) {
this.timeout = timeout;
}
public String getTimeout() {
return this.timeout == null ? "" : this.timeout;
}

public void setImpersonation(String impersonation) {
this.impersonation = impersonation;
}
public String getImpersonation() {
return this.impersonation == null ? "" : this.impersonation;
}
public void setAppName(String appName) {
this.appName = appName;
}
public String getAppName() {
return this.appName == null ? "" : this.appName;
}
public void allocate(int nrfields) {
inputFields = new PentahoGoogleSheetsPluginInputFields[nrfields];
}




@Override
public Object clone() {
Expand All @@ -157,6 +188,9 @@ public Object clone() {
retval.setJsonCredentialPath(this.jsonCredentialPath);
retval.setSpreadsheetKey(this.spreadsheetKey);
retval.setWorksheetId(this.worksheetId);
retval.setTimeout(this.timeout);
retval.setImpersonation(this.impersonation);
retval.setAppName(this.appName);
retval.setSampleFields(this.sampleFields);
for ( int i = 0; i < nrKeys; i++ ) {
retval.inputFields[i] = (PentahoGoogleSheetsPluginInputFields) inputFields[i].clone();
Expand All @@ -171,6 +205,9 @@ public String getXML() throws KettleException {
xml.append(XMLHandler.addTagValue("worksheetId", this.worksheetId));
xml.append(XMLHandler.addTagValue("spreadsheetKey", this.spreadsheetKey));
xml.append(XMLHandler.addTagValue("jsonCredentialPath", this.jsonCredentialPath));
xml.append(XMLHandler.addTagValue("timeout", this.timeout));
xml.append(XMLHandler.addTagValue("impersonation", this.impersonation));
xml.append(XMLHandler.addTagValue("appName", this.appName));
String tmp="100";
if(this.sampleFields!=null){
xml.append(XMLHandler.addTagValue("sampleFields", this.sampleFields.toString()));
Expand Down Expand Up @@ -207,6 +244,9 @@ public void loadXML(Node stepnode, List<DatabaseMeta> databases, IMetaStore meta
this.worksheetId = XMLHandler.getTagValue(stepnode, "worksheetId");
this.spreadsheetKey = XMLHandler.getTagValue(stepnode, "spreadsheetKey");
this.jsonCredentialPath = XMLHandler.getTagValue(stepnode, "jsonCredentialPath");
this.timeout = XMLHandler.getTagValue(stepnode, "timeout");
this.impersonation = XMLHandler.getTagValue(stepnode, "impersonation");
this.appName = XMLHandler.getTagValue(stepnode, "appName");
String tmp=XMLHandler.getTagValue(stepnode, "sampleField");
if(tmp!=null && !tmp.isEmpty()){
this.sampleFields = Integer.parseInt(tmp);
Expand Down Expand Up @@ -251,7 +291,11 @@ public void readRep(Repository rep, IMetaStore metaStore, ObjectId id_step, List
this.worksheetId = rep.getStepAttributeString(id_step, "worksheetId");
this.spreadsheetKey = rep.getStepAttributeString(id_step, "spreadsheetKey");
this.jsonCredentialPath = rep.getStepAttributeString(id_step, "jsonCredentialPath");
this.timeout = rep.getStepAttributeString(id_step, "timeout");
this.impersonation = rep.getStepAttributeString(id_step, "impersonation");
this.appName = rep.getStepAttributeString(id_step, "appName");
this.sampleFields =(int) rep.getStepAttributeInteger(id_step, "sampleFields");
this.appName = rep.getStepAttributeString(id_step, "appName");
int nrfields = rep.countNrStepAttributes(id_step, "field_name");

allocate(nrfields);
Expand Down Expand Up @@ -287,7 +331,9 @@ public void saveRep(Repository rep, IMetaStore metaStore, ObjectId id_transforma
rep.saveStepAttribute(id_transformation, id_step, "worksheetId", this.worksheetId);
rep.saveStepAttribute(id_transformation, id_step, "jsonCredentialPath", this.jsonCredentialPath);
rep.saveStepAttribute(id_transformation, id_step, "sampleFields", this.sampleFields.toString());

rep.saveStepAttribute(id_transformation, id_step, "timeout", this.timeout);
rep.saveStepAttribute(id_transformation, id_step, "impersonation", this.impersonation);
rep.saveStepAttribute(id_transformation, id_step, "appName", this.appName);
int nrfields = rep.countNrStepAttributes(id_step, "field_name");

for ( int i = 0; i < inputFields.length; i++ ) {
Expand Down
Loading

0 comments on commit c9282cf

Please sign in to comment.