Spring batch with google spanner with no in memory database - spring

Question

I'm thinking to implement the spring batch job with Google Spanner as a database, but spring batch expects standard databases only, I don't want to do this with the in-memory database/external, I wanted to store all the job metadata in Google Spanner, how can I implement this? Is there any inputs from experts who have implemented with GCP spanner?
I referred this answer Data Source for GCP Spanner
got the below error
Caused by: java.lang.IllegalArgumentException: DatabaseType not found for product name: [Google Cloud Spanner]
at org.springframework.batch.support.DatabaseType.fromProductName(DatabaseType.java:84) ~[spring-batch-infrastructure-4.2.1.RELEASE.jar:4.2.1.RELEASE]
at org.springframework.batch.support.DatabaseType.fromMetaData(DatabaseType.java:123) ~[spring-batch-infrastructure-4.2.1.RELEASE.jar:4.2.1.RELEASE]
at org.springframework.batch.core.repository.support.JobRepositoryFactoryBean.afterPropertiesSet(JobRepositoryFactoryBean.java:183) ~[spring-batch-core-4.2.1.RELEASE.jar:4.2.1.RELEASE]
at org.springframework.boot.autoconfigure.batch.BasicBatchConfigurer.createJobRepository(BasicBatchConfigurer.java:129) ~[spring-boot-autoconfigure-2.2.2.RELEASE.jar:2.2.2.RELEASE]
at org.springframework.boot.autoconfigure.batch.BasicBatchConfigurer.initialize(BasicBatchConfigurer.java:97) ~[spring-boot-autoconfigure-2.2.2.RELEASE.jar:2.2.2.RELEASE]
... 26 common frames omitted
configuration code is below,
package io.spring.batchdemo.config;
import javax.annotation.PostConstruct;
import javax.sql.DataSource;
import org.springframework.batch.core.configuration.annotation.BatchConfigurer;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.explore.support.JobExplorerFactoryBean;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.launch.support.SimpleJobLauncher;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean;
import org.springframework.boot.autoconfigure.batch.BasicBatchConfigurer;
import org.springframework.boot.autoconfigure.batch.BatchProperties;
import org.springframework.boot.autoconfigure.transaction.TransactionManagerCustomizers;
import org.springframework.boot.context.properties.PropertyMapper;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import org.springframework.transaction.PlatformTransactionManager;
#Configuration
public class BatchConfig2 implements BatchConfigurer {
private final BatchProperties properties;
private PlatformTransactionManager transactionManager;
private final TransactionManagerCustomizers transactionManagerCustomizers;
private JobRepository jobRepository;
private JobLauncher jobLauncher;
private JobExplorer jobExplorer;
/**
* Create a new {#link BasicBatchConfigurer} instance.
* #param properties the batch properties
* #param dataSource the underlying data source
* #param transactionManagerCustomizers transaction manager customizers (or
* {#code null})
*/
protected BatchConfig2(BatchProperties properties,
TransactionManagerCustomizers transactionManagerCustomizers) {
this.properties = properties;
this.transactionManagerCustomizers = transactionManagerCustomizers;
}
#Override
public JobRepository getJobRepository() {
return this.jobRepository;
}
#Override
public PlatformTransactionManager getTransactionManager() {
return this.transactionManager;
}
#Override
public JobLauncher getJobLauncher() {
return this.jobLauncher;
}
#Override
public JobExplorer getJobExplorer() throws Exception {
return this.jobExplorer;
}
#PostConstruct
public void initialize() {
try {
this.transactionManager = buildTransactionManager();
this.jobRepository = createJobRepository();
this.jobLauncher = createJobLauncher();
this.jobExplorer = createJobExplorer();
}
catch (Exception ex) {
throw new IllegalStateException("Unable to initialize Spring Batch", ex);
}
}
protected JobExplorer createJobExplorer() throws Exception {
PropertyMapper map = PropertyMapper.get();
JobExplorerFactoryBean factory = new JobExplorerFactoryBean();
factory.setDataSource(spannerDataSource());
map.from(this.properties::getTablePrefix).whenHasText().to(factory::setTablePrefix);
factory.afterPropertiesSet();
return factory.getObject();
}
protected JobLauncher createJobLauncher() throws Exception {
SimpleJobLauncher jobLauncher = new SimpleJobLauncher();
jobLauncher.setJobRepository(getJobRepository());
jobLauncher.afterPropertiesSet();
return jobLauncher;
}
protected JobRepository createJobRepository() throws Exception {
JobRepositoryFactoryBean factory = new JobRepositoryFactoryBean();
PropertyMapper map = PropertyMapper.get();
map.from(spannerDataSource()).to(factory::setDataSource);
map.from(this::determineIsolationLevel).whenNonNull().to(factory::setIsolationLevelForCreate);
map.from(this.properties::getTablePrefix).whenHasText().to(factory::setTablePrefix);
map.from(this::getTransactionManager).to(factory::setTransactionManager);
factory.afterPropertiesSet();
factory.setDatabaseType("spanner");//which datatype to set?, here is the error,Caused by: java.lang.IllegalArgumentException: DatabaseType not found for product name: [Google Cloud Spanner]
return factory.getObject();
}
/**
* Determine the isolation level for create* operation of the {#link JobRepository}.
* #return the isolation level or {#code null} to use the default
*/
protected String determineIsolationLevel() {
return null;
}
protected PlatformTransactionManager createTransactionManager() {
return new DataSourceTransactionManager(spannerDataSource());
}
private PlatformTransactionManager buildTransactionManager() {
PlatformTransactionManager transactionManager = createTransactionManager();
if (this.transactionManagerCustomizers != null) {
this.transactionManagerCustomizers.customize(transactionManager);
}
return transactionManager;
}
#Bean
public DataSource spannerDataSource() {
DriverManagerDataSource dataSource = new DriverManagerDataSource();
dataSource.setDriverClassName("com.google.cloud.spanner.jdbc.JdbcDriver");
dataSource.setUrl("jdbc:cloudspanner:/projects/suresh-project-261506/instances/suresh-spanner/databases/spanner?credentials=C:\\Users\\skengab\\AppData\\Roaming\\gcloud\\application_default_credentials.json");
return dataSource;
}
}
pom.xml is here
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.2.RELEASE</version>
<relativePath /> <!-- lookup parent from repository -->
</parent>
<groupId>io.spring</groupId>
<artifactId>batch-demo</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>batch-demo</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
<spring-cloud.version>Hoxton.RELEASE</spring-cloud.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-batch</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-gcp-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-gcp-starter-data-spanner</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-gcp-starter-storage</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-spanner-jdbc</artifactId>
<version>1.9.0</version>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.batch</groupId>
<artifactId>spring-batch-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>${spring-cloud.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

Answer 1

Spring Batch only has official support for a fixed set of databases. Google Cloud Spanner is not one of them. If you want to use Spring Batch with Spanner, you will need to set the database type manually to one of the supported databases. Spring Batch will then generate queries and other commands based on that setting, meaning that there is a chance that you could get errors regarding queries that are not compatible with Spanner.
If you can live with the above limitations, I would recommend trying to set the database type to POSTGRES, i.e. change the following line from
factory.setDatabaseType("spanner");//which datatype to set?, here is the error,Caused by: java.lang.IllegalArgumentException: DatabaseType not found for product name: [Google Cloud Spanner]
Into:
factory.setDatabaseType("POSTGRES");