Getting Spring & Hive integration working wasn't a breeze, but I got it. Here is what I had to do:
First, a typical BookRepository class:
package com.noushin.spring.ht.dao;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.hadoop.hive.HiveTemplate;
import org.springframework.stereotype.Repository;
import java.util.List;
/**
* This class handles accessing books in Hadoop using Hive.
*
* @author nbashir
*
*/
@Repository
public class BookRepositroy {
@Autowired
private HiveTemplate hiveTemplate;
public void showTables() {
List<String> tables = hiveTemplate.query("show tables;");
System.out.println("tables size: " + tables.size());
}
public Long count() {
return hiveTemplate.queryForLong("select count(*) from books;");
}
}
Next a typical Service layer class:
package com.noushin.spring.ht.service;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.noushin.spring.ht.dao.BookRepositroy;
/**
* This class handles any business logic related to handling books.
*
* @author nbashir
*
*/
@Component
public class BookService {
@Autowired
BookRepositroy bookRepo;
public Long count() {
return bookRepo.count();
}
public void showTables() {
bookRepo.showTables();
}
}
Here is the pom file you need:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.noushin.spring</groupId>
<artifactId>ht</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>ht</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<spring.hadoop.version>1.0.2.RELEASE</spring.hadoop.version>
<hadoop.version>1.2.1</hadoop.version>
<hive.version>0.10.0</hive.version>
<spring.version>4.0.3.RELEASE</spring.version>
</properties>
<dependencies>
<!-- Spring Data -->
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-hadoop</artifactId>
<version>${spring.hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-context-support</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>${hadoop.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-builtins</artifactId>
<version>${hive.version}</version>
<scope>runtime</scope>
</dependency>
</dependencies>
</project>
You application-context.xml:
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
xmlns:hdp="http://www.springframework.org/schema/hadoop" xmlns:c="http://www.springframework.org/schema/c"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd">
<context:property-placeholder location="hive.properties" />
<!-- Activate annotation configured components -->
<context:annotation-config />
<!-- Scan components for annotations within the configured package -->
<context:component-scan base-package="com.noushin.spring.ht" />
<hdp:hive-client-factory id="hiveClientFactory"
host="${hive.host}" port="${hive.port}">
<hdp:script>
ADD JARS /usr/lib/hive/lib/books.jar;
</hdp:script>
</hdp:hive-client-factory>
<hdp:hive-template id="hiveTemplate" />
</beans>
Your hive.properties:
hive.host=your-hive-server
hive.port=10000
hive.url=jdbc:hive://${hive.host}:${hive.port}/default
And finally a main class to start and run your app:
package com.noushin.spring.ht;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import com.noushin.spring.ht.service.BookService;
/**
* Main class to demonstrate accessing MongoDB with Spring Data and MongoTemplate.
*
* @author nbashir
*
*/
public class HadoopMain {
public static void main(String[] args) {
try {
ApplicationContext ctx = new ClassPathXmlApplicationContext("application-context.xml");
if (ctx != null) {
BookService service = ctx.getBean(BookService.class);
service.showTables();
Long count = service.count();
System.out.println("result : " + count);
}
}
catch (Exception ex) {
System.out.println("HadoopMain encountered an error and ended.");
}
}
}
