Gather and index content from an unsupported data source

void gather(PluginGatherContext pluginGatherContext, PluginStore store, FileScanner fileScanner) throws Exception;

void gather(PluginGatherContext pluginGatherContext, PluginStore store) throws Exception;

package com.example.mockgathering;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Stack;

import java.net.URI;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import com.example.mockgathering.remotefetch.RealRemoteFetcher;
import com.example.mockgathering.remotefetch.RemoteFetcher;
import com.funnelback.plugin.gatherer.PluginGatherContext;
import com.funnelback.plugin.gatherer.PluginGatherer;
import com.funnelback.plugin.gatherer.PluginStore;

/**
* The gather here will use the object in field "remoteFetcher" for actually
* fetching documents. The test for this class will show how it is possible to
* change the implementation of "remoteFetcher" such that it uses our own dummy
* mock class which does not make real HTTP requests.
*/
public class MockGatheringPluginGatherer implements PluginGatherer {

    private static final Logger log = LogManager.getLogger(MockGatheringPluginGatherer.class);

    // By default we use the real fetcher, under test will we change this.
    public RemoteFetcher remoteFetcher = new RealRemoteFetcher();

    @Override
    public void gather(PluginGatherContext pluginGatherContext, PluginStore store, FileScanner fileScanner) throws Exception {

        // Get the security-token from config, this will be supplied in each request.
        Map<String, String> headers = new HashMap<>();
        headers.put("security-token", pluginGatherContext.getConfigSetting(PluginUtils.KEY_PREFIX + "security-token"));

        Stack<URI> urlsToFetch = new Stack<>();

        // Get the start URLs which is a space separated list.
        Arrays.asList(pluginGatherContext.getConfigSetting(PluginUtils.KEY_PREFIX+"start-urls").split(" "))
            .stream()
            .filter(s -> !s.isBlank()) // remove blank/empty strings.
            .map(URI::create) // convert all of the strings to URLs
            .forEach(urlsToFetch::add); // add all the urls to the urlsToFetch array list.

        while(!urlsToFetch.isEmpty()) {
            ArrayListMultimap<String, String> metadata = ArrayListMultimap.create();

            URI url = urlsToFetch.pop();
            String fetchedContent = remoteFetcher.get(url, headers);
            // Scan document for viruses
            if (virusScanner.checkbytes(fetchedContent.getBytes())) {
                // Store the document
                store.store(
                        url,
                        fetchedContent.getBytes(StandardCharsets.UTF_8),
                        metadata
                );

                List<URI> moreUrlsToFetch = new FetchedResultParser().parseFetchedContent(url, fetchedContent, store);
                urlsToFetch.addAll(moreUrlsToFetch);
            } else {
                throw new RuntimeException("File " + url + " has failed virus scanning");
            }
        }
        log.debug("Gathering documents");
    }
}

package com.example.mockgathering;

import java.util.HashMap;
import java.util.Map;

import org.junit.Assert;
import org.junit.Test;

import java.net.URI;

import com.example.mockgathering.remotefetch.RemoteFetcher;
import com.funnelback.plugin.gatherer.mock.MockPluginGatherContext;
import com.funnelback.plugin.gatherer.mock.MockPluginStore;
import com.funnelback.plugin.gatherer.mock.MockFileScanner;

public class MockGatheringPluginGathererTest {

    /**
     * This is our mock remote fetcher, this is what we will use to stop making
     * HTTP requests in our test.
     *
     * This could have been in its own file like a normal class.
     */
    public static class MockRemoteFetcher implements RemoteFetcher {

        /**
         * When a URI is requested we will look for it in this map.
         */
        public Map<URI, String> uriToResult = new HashMap<>();

        @Override
        public String get(URI url, Map<String, String> headers) {
            // In our mock fetcher we ensure that the security-token is set.
            Assert.assertEquals("The security token was not set.", "password2", headers.get("security-token"));

            if(uriToResult.containsKey(url)) {
                return uriToResult.get(url);
            }

            throw new RuntimeException("404 could not find: " + url);
        }
    }

    /**
     * This shows how we mock making real web requests.
     */
    @Test
    public void testCustomGatherPlugin() throws Exception {
        MockPluginGatherContext mockContext = new MockPluginGatherContext();
        MockPluginStore mockStore = new MockPluginStore();
        MockFileScanner mockFileScanner = new MockFileScanner();

        // Configure the required config settings for the test:
        mockContext.setConfigSetting("plugin.mock-gathering.security-token", "password2");
        mockContext.setConfigSetting("plugin.mock-gathering.start-urls", "http://example.com/1");

        // Here we also create our mock RemoteFetcher, this way we don't make HTTP requests in our test.
        MockRemoteFetcher mockRemoteFetcher = new MockRemoteFetcher();

        // Lets add some data to our remote fetcher:
        mockRemoteFetcher.uriToResult.put(URI.create("http://example.com/1"), "{\n" +
            "   \"content\":\"hello1\",\n" +
            "   \"next\":\"http://example.com/2\"\n" +
            "}");

        mockRemoteFetcher.uriToResult.put(URI.create("http://example.com/2"), "{\n" +
            "   \"content\":\"hello2\",\n" +
            "   \"next\": null\n" +
            "}");

        // Set file scanner to return valid file
        mockFileScanner.setCheckResults(true);

        MockGatheringPluginGatherer underTest = new MockGatheringPluginGatherer();

        // It is important to actually tell our gather which RemoteFetcher we want it to use.
        // use our mock RemoteFetcher.
        underTest.remoteFetcher = mockRemoteFetcher;

        // Now run it, it will use our movk remote fetcher so we can test it without needing a real server.
        underTest.gather(mockContext, mockStore, mockFileScanner);

        Assert.assertEquals("Check how many documents were stored.", 2, mockStore.getStored().size());
    }
}