Sunday, June 16, 2013

Spring 4 & REST

In this blog I will show you how to use Spring to implement RESTful web services. As with any other Spring MVC application, you will need a controller class that is used to expose your web service methods.

Here is an example:
package com.noushin.spring.ws.controller;

import java.util.List;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;

import com.noushin.spring.ws.model.Student;
import com.noushin.spring.ws.service.StudentService;

/**
 * 
 * This class is used to demonstrate how Spring is used to implement RESTful web services.
 * 
 * @author nbashir
 *
 */
@Controller
@RequestMapping("/student")
public class StudentController {

   @Autowired
   StudentService studentService;
   
   /**
    * To test: curl -i -v -X POST -H "Content-Type: application/json;charset=UTF-8" http://localhost:8080/ws/student/add -d '{"name":"John Wait", "grade":1}'
    *  
    * @param student JSON representation of student to be added
    * 
    */
   @RequestMapping(value="/add", method=RequestMethod.POST, consumes="application/json;charset=UTF-8", produces="application/json;charset=UTF-8")
   @ResponseBody
   public Student addStudent(@RequestBody Student student) {
      return studentService.addStudent(student);
   }
   
   /**
    * To test:  curl -i -v -X GET -H "Content-Type: application/json;charset=UTF-8" http://localhost:8080/ws/student/get/16a61b8e-f6ae-459f-bb92-164bd7b72d07
    * 
    * @param id Student ID
    * @return A student in JSON format
    */
   @RequestMapping(value="/get/{id}", method=RequestMethod.GET, produces="application/json;charset=UTF-8")
   @ResponseBody
   public Student getStudent(@PathVariable String id) {
      return studentService.getStudent(id);
   }
   
   /**
    * To test:  curl -i -v -X GET -H "Content-Type: application/json;charset=UTF-8" http://localhost:8080/ws/student/getall
    * 
    * @return A list of all students in JSON format
    */
   @RequestMapping(value="/getall", method=RequestMethod.GET, produces="application/json;charset=UTF-8")
   public @ResponseBody List<Student> getStudents() {
      return studentService.getStudents();
   }
   
   /**
    * To test: curl -i -v -X GET http://localhost:8080/ws/student/testexception
    * Only for demonstration purposes.
    * 
    * @throws Exception 
    */
   @RequestMapping(value="/testexception", method=RequestMethod.GET)
   public void testException() throws Exception {
      System.out.println("Service is available and throws exception.");
      studentService.throwException();
   }
   
   /**
    * Exception Handler for this service.
    * 
    * @param exception Use a different subclass of Exception, if you need a more refined exception handling.
    * @return The same student object populated with info on the exception. 
    */
   @ExceptionHandler(Exception.class)
   public @ResponseBody String handleMyException(Exception exception) {
      return (new String("Error : " + exception.getMessage()));
   }
}
We will need a service class to support our controller methods. Note how service class is accessed in the controller by Spring Autowired feature.
package com.noushin.spring.ws.service;

import java.util.List;
import java.util.UUID;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.noushin.spring.ws.dao.StudentDao;
import com.noushin.spring.ws.model.Student;

/**
 * 
 * This class is used to demonstrate how Spring is used to implement RESTful web services.
 * 
 * @author nbashir
 * 
 */
@Component
public class StudentService {

   @Autowired
   StudentDao studentDao;

   public Student addStudent(Student student) {
      student.setId(UUID.randomUUID().toString());
      studentDao.addStudent(student);
      return student;
   }

   public Student getStudent(String id) {
      return studentDao.getStudent(id);
   }

   public List<Student> getStudents() {
      return studentDao.getStudents();
   }

   public void updateStudent(Student student) {
      studentDao.updateStudent(student);
   }

   public void deleteStudent(String id) {
      studentDao.deleteStudent(id);
   }

   public void throwException()  throws Exception {
      throw new Exception("This is an artificial Exception to test WS exception handling.");
   }
}
In a typical application, you would need a data layer that manages your data. Here is a typical Data Access Object (DAO) class. Note in this example, I am just using a local static Map to hold the data. In real world applications, you would be accessing some database or similar technology.
package com.noushin.spring.ws.dao;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import org.springframework.stereotype.Component;

import com.noushin.spring.ws.model.Student;

/**
 * 
 * This class is used to demonstrate how Spring is used to implement RESTful web services.
 * 
 * @author nbashir
 * 
 */
@Component
public class StudentDao {

   private static Map<String, Student> students = new HashMap<String, Student>();

   public void addStudent(Student student) {
      students.put(student.getId(), student);
   }

   public Student getStudent(String id) {
      return students.get(id);
   }

   public List<Student> getStudents() {
      return new ArrayList<Student>(students.values());
   }

   public void updateStudent(Student student) {
      deleteStudent(student.getId());
      addStudent(student);
   }

   public void deleteStudent(String id) {
      students.remove(id);
   }
}
And finally the business object you are trying to manage:
package com.noushin.spring.ws.model;

/**
 * 
 * This class is used to demonstrate how Spring is used to implement RESTful web services.
 * 
 * @author nbashir
 * 
 */
public class Student {

   private String id;
   private int grade;
   private String name;

   /**
    * Need this constructor for JSON mapping
    */
   public Student() {
      super();
   }

   public Student(int grade, String name) {
      super();
      this.grade = grade;
      this.name = name;
   }

   public Student(String id, int grade, String name) {
      super();
      this.id = id;
      this.grade = grade;
      this.name = name;
   }

   public String getId() {
      return id;
   }

   public void setId(String id) {
      this.id = id;
   }

   public int getGrade() {
      return grade;
   }

   public void setGrade(int grade) {
      this.grade = grade;
   }

   public String getName() {
      return name;
   }

   public void setName(String name) {
      this.name = name;
   }
}

To run your application as a web based application, you'd need a web.xml and servlet configuration file. Add the following files to your application WEB-INF directory. Example:

~/workspace/ws/src/main/webapp/WEB-INF/web.xml
~/workspace/ws/src/main/webapp/WEB-INF/ws-servlet.xml
web.xml:
<?xml version="1.0" encoding="ISO-8859-1"?>
<web-app version="3.0" metadata-complete="true"
    xmlns="http://java.sun.com/xml/ns/javaee" 
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd">

    <display-name>Spring WS Demo</display-name>
    <description>Spring WS Demo</description>

    <servlet>
        <servlet-name>ws</servlet-name>
        <servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>
        <load-on-startup>1</load-on-startup>
    </servlet>

    <servlet-mapping>
        <servlet-name>ws</servlet-name>
        <url-pattern>/</url-pattern>
    </servlet-mapping>
    
    <context-param>
        <param-name>contextConfigLocation</param-name>
        <param-value>/WEB-INF/ws-servlet.xml</param-value>
    </context-param>
 
     <context-param>
        <param-name>log4jConfigLocation</param-name>
        <param-value>/WEB-INF/classes/log4j.properties</param-value>
    </context-param>
    
    <listener>
        <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
    </listener>
   
</web-app>
ws-servlet.xml:
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xmlns:p="http://www.springframework.org/schema/p"
    xmlns:jms="http://www.springframework.org/schema/jms"
    xmlns:amq="http://activemq.apache.org/schema/core"
    xmlns:mvc="http://www.springframework.org/schema/mvc"
    xmlns:context="http://www.springframework.org/schema/context"
    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.2.xsd
                        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.2.xsd
                        http://www.springframework.org/schema/mvc http://www.springframework.org/schema/mvc/spring-mvc-3.2.xsd">


    <context:component-scan base-package="com.noushin.spring.ws" />
    <mvc:annotation-driven />
       
</beans>
Since I am using Maven to manage my application build and dependecies, here is the pom.xml you need:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>
    <groupId>com.noushin.spring</groupId>
    <artifactId>ws</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <packaging>war</packaging>
    <name>ws</name>
    <url>http://maven.apache.org</url>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <junit.version>4.10</junit.version>
        <log4j.version>1.2.17</log4j.version>
        <spring.version>4.1.0.RELEASE</spring.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>${junit.version}</version>
            <scope>test</scope>
        </dependency>
        
        <!-- Logging -->
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>${log4j.version}</version>
        </dependency>
        
        <!-- Spring Test -->
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-test</artifactId>
            <version>${spring.version}</version>
            <scope>test</scope>
        </dependency>
        
        <!-- Spring web -->
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-core</artifactId>
            <version>${spring.version}</version>
        </dependency>

        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-web</artifactId>
            <version>${spring.version}</version>
        </dependency>

        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-webmvc</artifactId>
            <version>${spring.version}</version>
        </dependency>

        <!-- Jackson mapper is used by spring to convert Java POJOs to JSON strings. -->
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.2.3</version>
        </dependency>
        
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-war-plugin</artifactId>
                <version>2.1.1</version>
                <configuration>
                    <failOnMissingWebXml>false</failOnMissingWebXml>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-dependency-plugin</artifactId>
                <version>2.1</version>
                <executions>
                    <execution>
                        <phase>validate</phase>
                        <goals>
                            <goal>copy</goal>
                        </goals>
                        <configuration>
                            <outputDirectory>${endorsed.dir}</outputDirectory>
                            <silent>true</silent>
                            <artifactItems>
                                <artifactItem>
                                    <groupId>javax</groupId>
                                    <artifactId>javaee-endorsed-api</artifactId>
                                    <version>6.0</version>
                                    <type>jar</type>
                                </artifactItem>
                            </artifactItems>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

</project>

And since no application development is ever complete without proper unit testing, here are some basic method level testing of StudentService. Note, I already included the curl commands you need to test your web service methods from command line in Linux in StudentController class.

StudentServiceTest.java:
package com.noushin.spring.ws.service;

import static org.junit.Assert.*;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.UUID;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import com.noushin.spring.ws.model.Student;
import com.noushin.spring.ws.service.StudentService;

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
public class StudentServiceTest {

   @Autowired
   StudentService service;

   @Before
   public void setUp() throws Exception {
   }

   @After
   public void tearDown() throws Exception {
   }

   @Test
   public void testAddStudent() {
      Student student = service.addStudent(new Student(5, "John Wait"));
      Student addedStudent = service.getStudent(student.getId());
      assertTrue(addedStudent.getGrade() == student.getGrade());
   }

   @Test
   public void testDeleteStudent() {
      Student student = service.addStudent(new Student(5, "John Wait"));
      student = service.getStudent(student.getId());
      service.deleteStudent(student.getId());
      Student deletedStudent = service.getStudent(student.getId());
      assertTrue(deletedStudent == null);
   }

   @Test
   public void testGetStudents() {
      Student student = new Student(1, "Jane Porter");
      service.addStudent(student);
      
      student = new Student(2, "John Downy");
      service.addStudent(student);

      student = new Student(3, "Kate Summerfield");
      service.addStudent(student);

      ArrayList<Student> students = (ArrayList<Student>) service.getStudents();
      assertTrue(students != null);      
      Iterator<Student> it = (Iterator<Student>) students.iterator();
      while (it.hasNext()) {
         student = (Student) it.next();
         System.out.println(student.getGrade() + ":" + student.getName());
      }
   }

   @Test
   public void testUpdateStudent() {
      Student student = service.addStudent(new Student(1, "Jane Porter"));
      int grade1 = student.getGrade();
      System.out.println("Student >>> " + student.getId() + " - Name : " + student.getName() + " - Grade : " + student.getGrade());
      student.setGrade(2);
      Student updatedStudent = service.getStudent(student.getId());
      System.out.println("updatedStudent >>> " + updatedStudent.getId() + " - Name : " + updatedStudent.getName() + " - Grade : " + updatedStudent.getGrade());
      int grade2 = updatedStudent.getGrade();
      assertTrue(grade1 != grade2);
   }
}

Since I am using Spring Test framework, here is the context file you need to run your tests. Place the following file in
~workspace/ws/src/test/resources/com/noushin/spring/ws/service/StudentServiceTest-context.xml
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:amq="http://activemq.apache.org/schema/core" xmlns:context="http://www.springframework.org/schema/context"
    xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
                        http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
                        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">

    <context:component-scan base-package="com.noushin.spring.ws" />
    <context:annotation-config />

</beans>


One more note regarding debugging Spring web services. Add the following lines to your log4j.properties file. They will come handy when you get errors when invoking web service methods:
log4j.category.org.springframework.web.servlet.DispatcherServlet=DEBUG
log4j.category.org.springframework.web.servlet.mvc=TRACE

Wednesday, April 10, 2013

Reading & Writing Hadoop Sequence Files in Java

In this blog I'll show you how to write a simple hadoop client application in Java. This app will handle reading, writing and copying Hadoop Sequence Files on local or remote Hadoop file systems (HDFS).

1. HadoopClient.java
package com.noushin.hadoop.client;

import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;

/**
 * This class handles interactions with Hadoop.
 * 
 * @author nbashir
 * 
 */
@Component
public class HadoopClient {

   private static Configuration conf = new Configuration();
   private final static Logger logger = Logger.getLogger(HadoopClient.class);

   /**
    * Convert the lines of text in a file to binary and write to a Hadoop
    * sequence file.
    * 
    * @param dataFile File containing lines of text
    * @param sequenceFileName Name of the sequence file to create
    * @param hadoopFS Hadoop file system
    * 
    * @throws IOException
    */
   public static void writeToSequenceFile(File dataFile, String sequenceFileName, String hadoopFS) throws IOException {

      IntWritable key = null;
      BytesWritable value = null;

      conf.set("fs.defaultFS", hadoopFS);
      Path path = new Path(sequenceFileName);

      if ((conf != null) && (dataFile != null) && (dataFile.exists())) {
         SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(path),
               SequenceFile.Writer.compression(SequenceFile.CompressionType.RECORD, new GzipCodec()),
               SequenceFile.Writer.keyClass(IntWritable.class), SequenceFile.Writer.valueClass(BytesWritable.class));

         List<String> lines = FileUtils.readLines(dataFile);

         for (int i = 0; i < lines.size(); i++) {
            value = new BytesWritable(lines.get(i).getBytes());
            key = new IntWritable(i);
            writer.append(key, value);
         }
         IOUtils.closeStream(writer);
      }
   }

   /**
    * Read a Hadoop sequence file on HDFS.
    * 
    * @param sequenceFileName Name of the sequence file to read
    * @param hadoopFS Hadoop file system
    * 
    * @throws IOException
    */
   public static void readSequenceFile(String sequenceFileName, String hadoopFS) throws IOException {
      conf.set("fs.defaultFS", hadoopFS);
      Path path = new Path(sequenceFileName);
      SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(path));
      IntWritable key = (IntWritable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
      BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
      while (reader.next(key, value)) {
         logger.info("key : " + key + " - value : " + new String(value.getBytes()));
      }
      IOUtils.closeStream(reader);
   }

   /**
    * Copy a local sequence file to a remote file on HDFS.
    * 
    * @param from Name of the sequence file to copy
    * @param to Name of the sequence file to copy to
    * @param remoteHadoopFS HDFS host URI
    * 
    * @throws IOException
    */
   public static void copySequenceFile(String from, String to, String remoteHadoopFS) throws IOException {
      conf.set("fs.defaultFS", remoteHadoopFS);
      FileSystem fs = FileSystem.get(conf);

      Path localPath = new Path(from);
      Path hdfsPath = new Path(to);
      boolean deleteSource = true;

      fs.copyFromLocalFile(deleteSource, localPath, hdfsPath);
      logger.info("Copied SequenceFile from: " + from + " to: " + to);
   }

   /**
    * Print all the values in Hadoop HDFS configuration object.
    * 
    * @param conf
    */
   public static void listHadoopConfiguration(Configuration conf) {
      int i = 0;
      logger.info("------------------------------------------------------------------------------------------");
      Iterator iterator = conf.iterator();
      while (iterator.hasNext()) {
         i++;
         iterator.next();
         logger.info(i + " - " + iterator.next());
      }
      logger.info("------------------------------------------------------------------------------------------");
   }
}

2. HadoopClientTest.java
package com.noushin.hadoop.client;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
public class HadoopClientTest {

   @Autowired
   HadoopClient hadoopClient;

   String sequenceFileName = "/tmp/nb.sgz";
   String hadoopLocalFS = "file:///";
   String hadoopRemoteFS = "hdfs://stage-hadoop01:8020";


   @Test
   public void testConfig() {
      Configuration conf = new Configuration();
      HadoopClient.listHadoopConfiguration(conf);
   }

   @Test
   public void testWriteSequenceFile() {
      String dataFileName = "/tmp/test.txt";

      try {
         int numOfLines = 20;
         String baseStr = "....Test...";
         List<String> lines = new ArrayList<String>();
         for (int i = 0; i < numOfLines; i++)
            lines.add(i + baseStr + UUID.randomUUID());

         File dataFile = new File(dataFileName);
         FileUtils.writeLines(dataFile, lines, true);
         Thread.sleep(2000);
         HadoopClient.writeToSequenceFile(dataFile, sequenceFileName, hadoopLocalFS);
      }
      catch (IOException e) {
          e.printStackTrace();
      }
      catch (InterruptedException e) {
         e.printStackTrace();
      }
   }

   @Test
   public void testReadSequenceFile() {

      try {       
         HadoopClient.readSequenceFile(sequenceFileName, hadoopLocalFS);
      }
      catch (IOException e) {
         e.printStackTrace();
      }
   }

   @Test
   public void testCopySequenceFileToRemoteHDFS() {
      String tempFileName = "/tmp/local-test.txt";
      String sequenceFileName = "/tmp/seqfile-record-compressed.sgz";
      String hadoopLocalFS = "file:///";
      String hadoopRemoteFS = "hdfs://stage-hadoop01:8020";

      try {
         int numOfLines = 5;
         String baseStr = "....Test...";
         List<String> lines = new ArrayList<String>();
         for (int i = 0; i < numOfLines; i++)
            lines.add(i + baseStr + UUID.randomUUID());

         File dataFile = new File(tempFileName);
         FileUtils.writeLines(dataFile, lines, true);
         Thread.sleep(2000);
         HadoopClient.writeToSequenceFile(dataFile, sequenceFileName, hadoopLocalFS);
         HadoopClient.readSequenceFile(sequenceFileName, hadoopLocalFS);
         HadoopClient.copySequenceFile(sequenceFileName, sequenceFileName, hadoopRemoteFS);
         HadoopClient.readSequenceFile(sequenceFileName, hadoopRemoteFS);
      }
      catch (IOException e) {
          e.printStackTrace();
      }
      catch (InterruptedException e) {
         e.printStackTrace();
      }
   }   
}

3. pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.noushin.hadoop</groupId>
    <artifactId>client</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>hdpc</name>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <hadoop-client.version>2.0.0-cdh4.2.0</hadoop-client.version>
        <junit.version>4.10</junit.version>
        <log4j.version>1.2.17</log4j.version>
        <spring.version>3.2.0.RELEASE</spring.version>
    </properties>

    <dependencies>
        <!-- Test -->
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>${junit.version}</version>
            <scope>test</scope>
        </dependency>

        <!-- Hadoop -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop-client.version}</version>
            <scope>provided</scope>
        </dependency>

        <!-- Logging -->
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>${log4j.version}</version>
        </dependency>

        <!-- Spring -->
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-context</artifactId>
            <version>${spring.version}</version>
        </dependency>
        
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>org.springframework.test</artifactId>
            <version>${spring.version}</version>
            <scope>test</scope>
        </dependency>
    </dependencies>
</project>
If you are using Eclipse for development and testing like I do, you need to add the following step, so you can compress your sequence file using GZip.

If you notice, I am using Hadoop by Cloudera in my pom file. To use GZip, I need to add a native library to my development environment which is Ubuntu 12.10.
sudo apt-get update; sudo apt-get install hadoop

This will install Hadoop native libraries in /usr/lib/hadoop/lib/native. Now, In Eclipse, edit ->Properties->Java Build Path->Libraries->Maven Dependencies->Native library location, and set "Location Path" to /usr/lib/hadoop/lib/native.
Please make sure the version of Hadoop client dependecy you use in your pom file matches the version of Hadoop you downloaded to your system, otherwise you will get a run time error:
ERROR nativeio.NativeIO: Unable to initialize NativeIO libraries
To verify a sequence file was created on HDFS, log into one of your hadoop nodes and run this command:
hadoop fs -ls /tmp/nb.sgz
And, if you run into a problem and need to see what Hadoop is doing, turn on debugging for Hadoop classes by adding the following entry to your log4j.properties: #Turn on hadoop logging

log4j.logger.org.apache.hadoop=DEBUG
To run Hive: login using a hadoop user such as oozie_job, so that environment is set up.

$ sudo su - oozie_job
To use hive:

$ hive
now you can query data using sql like commands:

DESCRIBE my_transactions;

SELECT * FROM my_transactions WHERE year=2013 AND month=3 AND day=14; 
To see where a partition is pointing to:

DESCRIBE EXTENDED my_transactions PARTITION(year=2013, month=3, day=28);
To create a partition, so Hive can find data for its queries:

ALTER TABLE my_transactions ADD PARTITION(year=2013, month=3, day=26) LOCATION '/tmp/2013/03/26';
To drop a partition and point it to a new location:

ALTER TABLE my_transactions DROP PARTITION(year=2013, month=3, day=26);

Tuesday, February 19, 2013

Oracle VM on Ubuntu

Every time I upgrade my Ubuntu, Oracle VM stops working. So, here is what I need to do after each OS update:
sudo apt-get install dkms build-essential linux-headers-generic
sudo /etc/init.d/vboxdrv setup
And VM is back...

Sunday, January 27, 2013

Spring & ActiveMQ SSL-enabled

In this blog I will show you how to configure ActiveMQ to use SSL as its transport.

Follow steps 1-4 on http://activemq.apache.org/how-do-i-use-ssl.html.

If running ActiveMQ as a server:
cd ~/activemq
vi conf/activemq.xml

Modify <transportConnector> to use SSL.
Before:
<transportConnector name="openwire" uri="tcp://0.0.0.0:61616?maximumConnections=1000&amp;wireformat.maxFrameSize=104857600"/>
After:
<transportConnector name="openwire" uri="ssl://localhost:61616?maximumConnections=1000&amp;wireformat.maxFrameSize=104857600"/>

Add SSLContext to <broker>.
<sslContext>
    <sslContext 
      keyStore="/home/nbashir/broker.ks" keyStorePassword="your-password" 
      trustStore="/home/nbashir/client.ts" trustStorePassword="your-password" />
</sslContext>

If you are using spring, you need the following info in your application-context files for producer and consumer classes:

Producer  application-context.xml
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:amq="http://activemq.apache.org/schema/core" xmlns:context="http://www.springframework.org/schema/context"
    xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
                        http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
                        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">

    <context:component-scan base-package="com.noushin.spring.jms" />
    <context:annotation-config />

    <!-- ActiveMQ destinations to use -->
    <amq:queue id="destination" physicalName="TestQ" />
    
    <bean id="sslConnectionFactory" class="org.apache.activemq.ActiveMQSslConnectionFactory"
        p:brokerURL="ssl://localhost:61616"
        p:trustStore="client.ts"
        p:trustStorePassword="your-password" />
    
    <!-- Spring JMS Producer Configuration -->
    <bean id="jmsProducerTemplate" class="org.springframework.jms.core.JmsTemplate"
        p:connectionFactory-ref="sslConnectionFactory"
        p:defaultDestination-ref="destination"/>
        
</beans>

 Consumer applciation-context.xml
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:amq="http://activemq.apache.org/schema/core" 
    xmlns:context="http://www.springframework.org/schema/context"
    xmlns:jms="http://www.springframework.org/schema/jms" 
    xmlns:p="http://www.springframework.org/schema/p"
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
                        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
                        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
                        http://www.springframework.org/schema/jms http://www.springframework.org/schema/jms/spring-jms.xsd">

    <context:component-scan base-package="com.noushin.spring.jms" />
    <context:annotation-config />
    
    <bean id="sslConnectionFactory" class="org.apache.activemq.ActiveMQSslConnectionFactory"
        p:brokerURL="ssl://localhost:61616"
        p:trustStore="client.ts"
        p:trustStorePassword="your-password" />
    
   <!-- JMS Consumer Configuration -->        
    <jms:listener-container container-type="default" 
                            connection-factory="sslConnectionFactory"
                            acknowledge="auto">
        <jms:listener destination="TestQ" ref="messageConsumer" />
    </jms:listener-container>
    
</beans>


Alternatively, you can configure ActiveMQ to run as a embedded JMS provider within your app. For that add the following lines to your application's application-context.xml
<!-- SSL Context --> 
<amq:broker useJmx="true" persistent="false">
    <amq:sslContext>
        <amq:sslContext 
            keyStore="/home/nbashir/broker.ks"
            keyStorePassword="your-password" 
            trustStore="/home/nbashir/client.ts"
            trustStorePassword="your-password" />
    </amq:sslContext>
    <amq:transportConnectors>
        <amq:transportConnector uri="ssl://localhost:61616" />
    </amq:transportConnectors>
</amq:broker>

Monday, January 14, 2013

Spring 3 & JMS

Here is a quick way to develop a simple application using Jms.  You will need the following components:

  • Apache ActiveMQ
  • Spring
  • Java
  • Maven
 
Install ActiveMQ on Ubuntu. The latest download bundle is available at ActiveMQ download page.

Example (Running ActiveMQ 5.7.0 on Ubuntu):

1. Download apache-activemq-5.7.0-bin.tar.gz.

2. Untar the bundle:

tar zxvf apache-activemq-5.7.0-bin.tar.gz

3. Configure and start Activemq

cd apache-activemq-5.7.0
bin/activemq setup newConfig
bin/activemq start

4. Verify Activemq is running:

netstat -an |grep 61616
or
Go to Admin console by visiting http://localhost:8161/admin/

5. Create a basic Jms queue for testing:

Use Admin console.
Select Queues
Create a new queue called TestQ.

Once you have created a Jms queue, you need to write classes to produce and consume messages that are transmitted via the newly created queue.

We will utilize Spring to transmit messages over JMS queues. Since this type of delivery is point to point, you need message producers at one end and message consumers at the other end.

JMS Message Producer

1. Create a context file called: ~/workspace/jms/src/main/resources/application-context.xml.
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:amq="http://activemq.apache.org/schema/core" 
    xmlns:context="http://www.springframework.org/schema/context"
    xmlns:p="http://www.springframework.org/schema/p"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
                        http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
                        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">

    <context:component-scan base-package="com.noushin.spring.jms" />
    <context:annotation-config />

    <!-- ActiveMQ destinations to use -->
    <amq:queue id="destination" physicalName="TestQ" />
    
    <!-- ActiveMQ broker URL -->
    <amq:connectionFactory id="jmsFactory" brokerURL="tcp://localhost:61616" />

    <!-- Spring JMS ConnectionFactory -->
    <bean id="singleConnectionFactory" 
          class="org.springframework.jms.connection.SingleConnectionFactory"
          p:targetConnectionFactory-ref="jmsFactory"/>
    
    <!-- Spring JMS Producer Configuration -->
    <bean id="jmsProducerTemplate" class="org.springframework.jms.core.JmsTemplate"
        p:connectionFactory-ref="singleConnectionFactory"
        p:defaultDestination-ref="destination"/>
        
</beans>


2. Create a class that produces messages and sends them over Jms. Lets call it MessageProducer.
package com.noushin.spring.jms.producer;

import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.Session;
import javax.jms.TextMessage;

import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.jms.core.MessageCreator;
import org.springframework.stereotype.Component;

@Component
public class MessageProducer {

   final static Logger logger = Logger.getLogger(MessageProducer.class);

   @Autowired
   private JmsTemplate jmsTemplate;

   public void produce() throws Exception {
      
      if (jmsTemplate != null) {
         MessageCreator mc = new MessageCreator() {
            public Message createMessage(Session session) throws JMSException {
               try {
                  TextMessage message = session.createTextMessage("This is a message.");
                  return message;
               } 
                catch (JMSException je) {
                  logger.error("JMS Exception : ", je);
                  return null;
               }
            }
         };
         jmsTemplate.send(mc);
      }
   }
}

3. Here is the pom.xml to successfully run this example
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>
    <groupId>com.noushin.spring</groupId>
    <artifactId>jms</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <packaging>jar</packaging>
    <name>jms</name>
    <url>http://maven.apache.org</url>

    <properties>
        <activemq.version>5.2.0</activemq.version>
        <junit.version>4.10</junit.version>
        <log4j.version>1.2.17</log4j.version>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <spring.version>3.2.0.RELEASE</spring.version>
    </properties>

    <repositories>
        <repository>
            <id>springsource-repo</id>
            <name>SpringSource Repository</name>
            <url>http://repo.springsource.org/release</url>
        </repository>
    </repositories>

    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>${junit.version}</version>
            <scope>test</scope>
        </dependency>
                <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>${log4j.version}</version>
        </dependency>
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-context</artifactId>
            <version>${spring.version}</version>
        </dependency>
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-jms</artifactId>
            <version>${spring.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.activemq</groupId>
            <artifactId>activemq-core</artifactId>
            <version>${activemq.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.activemq</groupId>
            <artifactId>activemq-optional</artifactId>
            <version>${activemq.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.xbean</groupId>
            <artifactId>xbean-spring</artifactId>
            <version>3.7</version>
        </dependency>
    </dependencies>
</project>

4. After running MessageProducer.main, you should see a message added to TestQ queue.

5. Use ActiveMQ admin console to verify the above steps: http://localhost:8161/admin/queues.jsp

JMS Message Consumer

Now we need to write a class that consumes the messages in the queue waiting to be processed. In this example, I will create a second project.

1.   Create a context file called: ~/workspace/jmsc/src/main/resources/application-context.xml.
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:amq="http://activemq.apache.org/schema/core" 
    xmlns:context="http://www.springframework.org/schema/context"
    xmlns:jms="http://www.springframework.org/schema/jms" 
    xmlns:p="http://www.springframework.org/schema/p"
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
                        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
                        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
                        http://www.springframework.org/schema/jms http://www.springframework.org/schema/jms/spring-jms.xsd">

    <context:component-scan base-package="com.noushin.spring.jms" />
    <context:annotation-config />

    <!-- ActiveMQ destinations to use -->
    <amq:queue id="destination" physicalName="TestQ" />

    <!-- ActiveMQ broker -->
    <amq:connectionFactory id="jmsFactory" brokerURL="tcp://localhost:61616" />

   <!-- JMS Consumer Configuration -->
    <bean id="jmsConsumerConnectionFactory" 
          class="org.springframework.jms.connection.SingleConnectionFactory"
          p:targetConnectionFactory-ref="jmsFactory" />
        
    <jms:listener-container container-type="default" 
                            connection-factory="jmsConsumerConnectionFactory"
                            acknowledge="auto">
        <jms:listener destination="TestQ" ref="messageConsumer" />
    </jms:listener-container>

</beans>

2.  Create a class that consumes messages as they arrive on the queue. Let's call it MessageConsumer.
package com.noushin.spring.jms.consumer;

import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageListener;
import javax.jms.TextMessage;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;

@Component
public class MessageConsumer implements MessageListener {

   final static Logger logger = Logger.getLogger(MessageConsumer.class);

   private int numOfMessages = 0;

   public void onMessage(Message message) {
      try {
         numOfMessages++;
         if (message instanceof TextMessage) {
            TextMessage tm = (TextMessage) message;
            String msg = tm.getText();
            logger.info(">>>>>Processed message: " + msg + " - numOfMessages : " + numOfMessages);
         }
      } catch (JMSException e) {
         logger.error(e.getMessage(), e);
      }
   }
}

3. You can use the same pom file you used for MessageProducer. Make sure to change your project name in the pom file.

4. To test your app, write a JUnit
package com.noushin.spring.jmsc;

import org.junit.Test;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

public class AppTest {

   @Test 
   public void testApp() {
         ApplicationContext ctx = new ClassPathXmlApplicationContext("application-context.xml");
    }
}

5. As soon as application context is initialized, go to ActiveMQ admin console and notice the messages you produced in the first project are now removed from the queue. You should also see messages logging the results of executing onMessage method in MessageConsumer class.
2013-01-10 11:50:24,365 [org.springframework.jms.listener.DefaultMessageListenerContainer#0-1] INFO  com.noushin.spring.jms.consumer.MessageConsumer - Processed message: this is a test. - numOfMessages : 2

6. That's all folks. Have fun with Jms :)

Saturday, January 12, 2013

Spring & Testing

I think unit testing Spring components is really cool. You can have a separate application context just for testing purposes, which will not conflict with your application's runtime context when developing.

Here is a couple of steps you need to take to test your components using Spring testing.

1. Add the following dependency to your pom file:
<dependency>
    <groupId>org.springframework</groupId>
    <artifactId>org.springframework.test</artifactId>
    <version>${spring.version}</version>
    <scope>test</scope>
</dependency>
where
    <spring.version>3.2.0.RELEASE</spring.version>

2. In your test/resources folder, your need to a create a package matching the your Test class, and create a context file with a name that matches your Test class name.

Here is an example:

Lets say you are testing a class called MessageProducer.
package com.noushin.spring.jms.producer;

import java.io.IOException;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.Session;
import javax.jms.TextMessage;

import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.jms.core.MessageCreator;
import org.springframework.stereotype.Component;

@Component
public class MessageProducer {

   final static Logger logger = Logger.getLogger(MessageProducer.class);

   @Autowired
   private JmsTemplate jmsTemplate;

   public void produce() throws Exception {
      
      if (jmsTemplate != null) {
         MessageCreator mc = new MessageCreator() {
            public Message createMessage(Session session) throws JMSException {
               try {
                  String jmsMessage = FileUtils.readFileToString(FileUtils.toFile(this.getClass().getResource("/jms-message.txt")));
                  TextMessage message = session.createTextMessage(jmsMessage);
                  logger.info(">>>>>Sending message: " + jmsMessage);
                  return message;
               } 
               catch (IOException ioe) {
                  logger.error("File not found : ", ioe);
                  return null;
               }
               catch (JMSException je) {
                  logger.error("JMS Exception : ", je);
                  return null;
               }
            }
         };
         jmsTemplate.send(mc);
      }
   }
}

3. Here is your JUnit test case, ~workspace/jms/src/test/java/com/noushin/spring/jms/producer/MessageProducerTest.java
package com.noushin.spring.jms.producer;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
public class MessageProducerTest {
   
   @Autowired
   protected MessageProducer producer;
   
   @Test
   public void testProduce() {
      try {
         producer.produce();
         assert(true);
      } 
      catch (Exception e) {
         e.printStackTrace();
      }
   }
}

4. Corresponding test context is at ~workspace/jms/src/test/resources/com/noushin/spring/jms/producer/MessageProducerTest-context.xml
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:amq="http://activemq.apache.org/schema/core" 
    xmlns:context="http://www.springframework.org/schema/context"
    xmlns:p="http://www.springframework.org/schema/p"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
                        http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd
                        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">

    <context:component-scan base-package="com.noushin.spring.jms" />
    <context:annotation-config />

    <!-- ActiveMQ destinations to use -->
    <amq:queue id="destination" physicalName="TestQ" />
    
    <!-- ActiveMQ broker URL -->
    <amq:connectionFactory id="jmsFactory" brokerURL="tcp://localhost:61616" />

    <!-- Spring JMS ConnectionFactory -->
    <bean id="singleConnectionFactory" 
          class="org.springframework.jms.connection.SingleConnectionFactory"
          p:targetConnectionFactory-ref="jmsFactory"/>
    
    <!-- Spring JMS Producer Configuration -->
    <bean id="jmsProducerTemplate" class="org.springframework.jms.core.JmsTemplate"
        p:connectionFactory-ref="singleConnectionFactory"
        p:defaultDestination-ref="destination"/>
        
</beans>

Alternatively, if there is a context configuration that can be reused by test classes in different packages, you can specify its location on classpath:
@ContextConfiguration(locations={"classpath:/com/noushin/spring/jms/producer/MessageProducerTest-context.xml"})

5. If in Eclipse, right click your test class and run it as JUnit Test. :)