Spring boot with Java API Client to Build and Execute Queries in Elasticsearch.

Spring boot with ES Java API Client to Build and Execute Queries in Elasticsearch.

Prerequisites:

Knowledge in Java, Spring boot, Elasticsearch, Kibana.

Concept:

The purpose of this blog is to present an idea for connecting, constructing queries, and querying Elasticsearch through Java applications.

What is Elasticsearch?

Elasticsearch is a distributed, free and open search and analytics engine for all types of data, including textual, numerical, geospatial, structured, and unstructured. It is the central component of the Elastic Stack, a collection of free and open tools for data ingestion, enrichment, storage, analysis, and visualization, and is known for its simple REST APIs, distributed nature, speed, and scalability.

Technologies used:

1. Elasticsearch 8.3.3
2. Spring boot 2.7.2
3. Java 1.8
4. Elasticsearch Java API client 7.17.5
5. Maven

Tools used:

1. Kibana 8.3.3
2. Postman

Note: The blog focuses only on a part of the CRUD operation. Click here for the complete source code with CRUD operations.

Project Structure:

Step 1: Create a Spring boot application using Spring Initalizr and select the dependencies as shown in the snapshot below.

Step 2: Add the additional dependencies given in the pom.xml file below.

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.7.2</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.poc.es</groupId>
    <artifactId>elasticsearch-springboot</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>elasticsearch-springboot</name>
    <description>Demo project for integrating elasticsearch with springboot</description>
    <properties>
        <java.version>1.8</java.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>co.elastic.clients</groupId>
            <artifactId>elasticsearch-java</artifactId>
            <version>7.17.5</version>
        </dependency>

        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.12.3</version>
        </dependency>

        <dependency>
            <groupId>jakarta.json</groupId>
            <artifactId>jakarta.json-api</artifactId>
            <version>2.0.1</version>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-devtools</artifactId>
            <scope>runtime</scope>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <configuration>
                    <excludes>
                        <exclude>
                            <groupId>org.projectlombok</groupId>
                            <artifactId>lombok</artifactId>
                        </exclude>
                    </excludes>
                </configuration>
                <version>${project.parent.version}</version>
            </plugin>
        </plugins>
    </build>

</project>

Step 3: Configuring Elasticsearch in Spring boot application.

application.yml

elastic:
  index: employees
es:
  hostname: localhost
  port: 9200
  username: admin
  password: password

ESRestClient.java

package com.poc.es.elasticsearchspringboot.config;

import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.ElasticsearchTransport;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import lombok.Getter;
import lombok.Setter;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;


@Configuration
@ConfigurationProperties("es")
@Getter
@Setter
public class ESRestClient {

    private String hostName;
    private int port;
    private String username;
    private String password;

    @Bean
    public ElasticsearchClient getElasticSearchClient() {

        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(username, password));

        RestClientBuilder builder = RestClient.builder(new HttpHost(hostName, port))
                .setHttpClientConfigCallback(httpClientBuilder ->
                        httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));

        // Create the low-level client
        RestClient restClient = builder.build();

        // Create the transport with a Jackson mapper
        ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper());

        // And create the API client
        return new ElasticsearchClient(transport);
    }
}

Choosing a Global Software Development Partner to Accelerate Your Digital Strategy

To be successful and outpace the competition, you need a software development partner that excels in exactly the type of digital projects you are now faced with accelerating, and in the most cost effective and optimized way possible.

Get the Guide

Step 4: Create Rest controller ESRestController.java

@Autowired
private ESService esService;

@PostMapping("/index/fetchWithMust")
public ResponseEntity<List<Employee>> fetchEmployeesWithMustQuery(@RequestBody Employee employeeSearchRequest) throws IOException {
    List<Employee> employees = esService.fetchEmployeesWithMustQuery(employeeSearchRequest);
    return ResponseEntity.ok(employees);
}

@PostMapping("/index/fetchWithShould")
public ResponseEntity<List<Employee>> fetchEmployeesWithShouldQuery(@RequestBody Employee employeeSearchRequest) throws IOException {
    List<Employee> employees = esService.fetchEmployeesWithShouldQuery(employeeSearchRequest);
    return ResponseEntity.ok(employees);
}

Step 5: Create a model Employee.java.

package com.poc.es.elasticsearchspringboot.model;

import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class Employee {
    private Long id;
    private String firstName;
    private String lastName;
    private String email;
    private String gender;
    private String jobTitle;
    private String phone;
    private Integer size;
}

Step 6: Create an interface ESService.java and ESServiceImpl.java.

@Service
public class ESServiceImpl implements ESService {
  
  @Autowired
  private ESClientConnector esClientConnector;

  @Override
  public List<Employee> fetchEmployeesWithMustQuery(Employee employee) throws IOException {
         return esClientConnector.fetchEmployeesWithMustQuery(employee);
  }

  @Override
  public List<Employee> fetchEmployeesWithShouldQuery(Employee employee) throws IOException {
         return esClientConnector.fetchEmployeesWithShouldQuery(employee);
  }
}

Step 7: Create a connector class that makes Elasticsearch API calls ESClientConnector.java.

@Value("${elastic.index}")
private String index;

@Autowired
private ElasticsearchClient elasticsearchClient;   

public List<Employee> fetchEmployeesWithMustQuery(Employee employee) throws IOException {
     List<Query> queries = prepareQueryList(employee);
     SearchResponse<Employee> employeeSearchResponse = elasticsearchClient.search(req->
             req.index(index)
                     .size(employee.getSize())
                     .query(query->
                             query.bool(bool->
                                     bool.must(queries))),
             Employee.class);

     return employeeSearchResponse.hits().hits().stream()
             .map(Hit::source).collect(Collectors.toList());
     }

 public List<Employee> fetchEmployeesWithShouldQuery(Employee employee) throws IOException {
     List<Query> queries = prepareQueryList(employee);
     SearchResponse<Employee> employeeSearchResponse = elasticsearchClient.search(req->
                     req.index(index)
                             .size(employee.getSize())
                             .query(query->
                                     query.bool(bool->
                                             bool.should(queries))),
             Employee.class);

     return employeeSearchResponse.hits().hits().stream()
             .map(Hit::source).collect(Collectors.toList());
     }

 private List<Query> prepareQueryList(Employee employee) {
      Map<String, String> conditionMap = new HashMap<>();
      conditionMap.put("firstName.keyword", employee.getFirstName());
      conditionMap.put("lastName.keyword", employee.getLastName());
      conditionMap.put("gender.keyword", employee.getGender());
      conditionMap.put("jobTitle.keyword", employee.getJobTitle());
      conditionMap.put("phone.keyword", employee.getPhone());
      conditionMap.put("email.keyword", employee.getEmail());

      return conditionMap.entrySet()
                          .stream()
                          .filter(entry->!ObjectUtils.isEmpty(entry.getValue()))
                          .map(entry->QueryBuilderUtils.termQuery(entry.getKey(), entry.getValue()))
                          .collect(Collectors.toList());
     }

Step 8: Create Util interface to build ES queries QueryBuilderUtils.java

package com.poc.es.elasticsearchspringboot.utils;

import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.elasticsearch._types.query_dsl.QueryVariant;
import co.elastic.clients.elasticsearch._types.query_dsl.TermQuery;

public interface QueryBuilderUtils {

    public static Query termQuery(String field, String value) {
        QueryVariant queryVariant = new TermQuery.Builder()
                .caseInsensitive(true)
                .field(field).value(value).build();
        return new Query(queryVariant);
    }
}

API Calls through Postman:

Fetch data from Elasticsearch using MUST clause:

Logs: Constructed ES query with “MUST” clause in Java

POST /employees/_search?typed_keys=true {
    "query": {
        "bool": {
            "must": [
                {
                    "term": {
                        "jobTitle.keyword": {
                            "value": "Senior Developer",
                            "case_insensitive": true
                        }
                    }
                },
                {
                    "term": {
                        "gender.keyword": {
                            "value": "Female",
                            "case_insensitive": true
                        }
                    }
                }
            ]
        }
    }
}

Fetch data from Elasticsearch using SHOULD clause:

Logs: Constructed ES query with “SHOULD” clause in Java

POST /employees/_search?typed_keys=true {
    "query": {
        "bool": {
            "should": [
                {
                    "term": {
                        "jobTitle.keyword": {
                            "value": "Senior Developer",
                            "case_insensitive": true
                        }
                    }
                },
                {
                    "term": {
                        "gender.keyword": {
                            "value": "Female",
                            "case_insensitive": true
                        }
                    }
                }
            ]
        }
    }
}

Project Github URL: https://github.com/sundharamurali/elasticsearch-springboot.git