Incredibly slow text search runtime [Optimization]

Question

What am i trying to do

I have a huge 8.5 GB text file containing 3 million lines in word format, followed by 300 numbers, for example:

word 0.056646 -0.0256464 0.05246 (etc.)

300 numbers per word form a vector that represents the word. I have 3 words with which I have to find the vector that most accurately represents the fourth word using the analogy model (I use addition, multiplication and direction).

To add, it will look like this:

Say you have vectors of vectors a, b and c, then I would do c - a + b. Then I repeat all 3 million lines and use the cosine semblance to find the fourth word d, looking for the maximum result. Thus, it looks like this: d = max (cos (d ', ca + b)), where d' stands for the word in the current line.

What is the problem

The above example presents one request. I have to complete a total of 20,000 queries. And I'm not just doing it for a similar addition model, but also for multiplication and direction. When I run my program, it is still trying to calculate the fourth word for the first similar model (addition) for the first query after 30 seconds! I really need optimization in my program.

-, 3 (3 ), , a, b c. System.nanoTime(), , 1,5 . 5 , 3.

, , , (, , API, ):

public class VectorCalculation {

    public static List<Double> plus(List<Double> v1, List<Double> v2){
        return operation(new Plus(), v1, v2);
    }

    public static List<Double> minus(List<Double> v1, List<Double> v2){
        return operation(new Minus(), v1, v2);
    }

    public static List<Double> operation(Operator op, List<Double> v1, List<Double> v2){
        if(v1.size() != v2.size())  throw new IllegalArgumentException("The dimension of the given lists are not the same.");
        List<Double> resultVector = new ArrayList<Double>();
        for(int i = 0; i < v1.size(); i++){
            resultVector.add(op.calculate(v1.get(i), v2.get(i)));
        }
        return resultVector;
    }
}

public interface Operator {
    public Double calculate(Double e1, Double e2);
}

public class Plus implements Operator {

    @Override
    public Double calculate(Double e1, Double e2) {
        return e1+e2;
    }
}

public class Minus implements Operator {

    @Override
    public Double calculate(Double e1, Double e2) {
        return e1-e2;
    }
}

:

public class Addition extends AnalogyModel {

    @Override
    double calculateWordVector(List<Double> a, List<Double> b, List<Double> c, List<Double> d) {
        //long startTime1 = System.nanoTime();
        List<Double> result = VectorCalculation.plus(VectorCalculation.minus(c, a), b);
        //long endTime1 = System.nanoTime() - startTime1;
        double result2 = cosineSimilarity(d, result);
        //long endTime2 = System.nanoTime() - startTime1;
        //System.out.println(endTime1 + "       |       " + endTime2);
        return result2;
    }

    Double cosineSimilarity(List<Double> v1, List<Double> v2){
        if(v1.size() != v2.size())  throw new IllegalArgumentException("Vector dimensions are not the same.");

        // find the dividend
        Double dividend = dotProduct(v1, v2);

        // find the divisor
        Double divisor = dotProduct(v1, v1) * dotProduct(v2, v2);
        if(divisor == 0)    divisor = 0.0001;   // safety net against dividing by 0.

        return dividend/divisor;
    }

    /**
     * @return  Returns the dot product of two vectors.
     */
    Double dotProduct(List<Double> v1, List<Double> v2){
        System.out.println(v1);
        Double result = 0.0;
        for(int i = 0; i < v1.size(); i++){
            result += v1.get(i)*v2.get(i);
        }
        return result;
    }
}

, , ( 0,1 ), 0,025 . , 2, , 0,005 . d ' 3 . 0,06 .

: , , , 5 + 3000000 * (0,025 + 0,005 + 0,06) = 270005 270 4,5 , ... , , 20000 , .

. , , , , . , ?

-

    /**
     * @param vocabularyPath    The path of the vector text file.
     * @param word              The word to find the vector for.
     * @return  Returns the vector of the given word as an array list.
     */
    List<Double> getStringVector(String vocabularyPath, String word) throws IOException{
        BufferedReader br = new BufferedReader(new FileReader(vocabularyPath));

        String input = br.readLine();
        boolean found = false;
        while(!found && input != null){
            if(input.contains(word))    found = true;
            else input = br.readLine();
        }

        br.close();
        if(input == null)   return null;
        else return getVector(input);
    }

    /**
     * @param inputLine A line from the vector text file.
     * @return  Returns the vector of the given line as an array list.
     */
    List<Double> getVector(String inputLine){
        String[] splitString = inputLine.split("\\s+");
        List<String> stringList = new ArrayList<>(Arrays.asList(splitString));
        stringList.remove(0); // remove the word at the front
        stringList.remove(stringList.size()-1); // remove the empty string at the end
        List<Double> vectorList = new ArrayList<>();
        for(String s : stringList){
            vectorList.add(Double.parseDouble(s));
        }
        return vectorList;
    }

Answer 1

: List<Double> Operator.

, 8 double (btw. float, , ), (, ). : , .

, N . , , .

, , double[]. .

operation -

public static void operationTo(double[] result, Operator op, double[] v1, double[] v2){
    int length = result.length;
    if(v1.length != length || v2.length != length) {
        throw new IllegalArgumentException("The dimension of the given lists are not the same.");
    }
    switch (op) { // use an enum
        case PLUS:
            for(int i = 0; i < length; i++) {
                result[i] = v1[i] + v2[i];
            }
        break;
        ...
    }
}

Word

- HashMap<String, double[]>, , . ( ). . , , a Map, 10 .

,

3M , . ArrayList . , . , , , ,

long index = Arrays.binarySeach(wordList, word);
randomAccessFile.seek(index * vectorLength * Double.SIZE / Byte.SIZE)

Answer 2

, 20000 3 300- ?

. , , , , .