Clojureでの one hot vectorとcategory vector

(ns test01.core
  (:require [clojure.data.csv :as csv]
           [clojure.java.io :as io]
           [clojure.string :as str])
  (:import (java.io File FileInputStream InputStreamReader BufferedReader StringReader
                    BufferedWriter OutputStreamWriter FileOutputStream)
           (org.apache.lucene.analysis.ja JapaneseAnalyzer JapaneseTokenizer)
           (org.apache.lucene.analysis.ja.tokenattributes PartOfSpeechAttribute)
           (org.apache.lucene.analysis.tokenattributes CharTermAttribute OffsetAttribute)
           (org.apache.lucene.analysis.util CharArraySet)))

;;csvからデータ読み込み(Read data from csv)
(def data-reads
  (with-open [reader (io/reader "data/in/test-data.csv")]  ;;read test-data from folder
    (doall
     (csv/read-csv reader))))

;;形態素解析関数(morphological function)
(defn morphological-analysis
  [src]
  (let [analyzer (JapaneseAnalyzer. nil
                                    JapaneseTokenizer/DEFAULT_MODE
                                    CharArraySet/EMPTY_SET
                                    #{})
        rdr (StringReader. src)]
    (with-open [ts (.tokenStream analyzer "field" rdr)]
      (let [^OffsetAttribute       offsetAtt (.addAttribute ts OffsetAttribute)
            ^PartOfSpeechAttribute posAtt    (.addAttribute ts PartOfSpeechAttribute)
            _       (.reset ts)
            surface #(subs src (.startOffset offsetAtt) (.endOffset offsetAtt))
            pos     #(.getPartOfSpeech posAtt)
            tokens  (->> #(if (.incrementToken ts)
                            [(surface) (pos)]
                            nil)
                         repeatedly
                         (take-while identity)
                         doall)
            _       (.end ts)]
        tokens))))

;;形態素解析の結果を(品詞,単語)の形に変形(Transform morphological analysis results into (part of speech, word) form)
(defn simple-morph [n]
  (map #(list (first (str/split (% 1) #"-")) (% 0))
       (morphological-analysis n)))

;;名詞抽出(pick up named entity)
(defn meisi-token [n]
  (keep
   #(cond
      (= "名詞" (first %)) (second %)
      :else nil)(simple-morph n)))
;;stop wordデータ(stop-word-num.txt等)を抜いて出現する単語辞書を作成(Create a dictionary of words that appear by extracting stop word data (stop-word-num.txt, etc.))
(def word-dic-raw3
  (->>
   (distinct (flatten (map #(meisi-token (nth (nth data-reads %) 1))(range (count data-reads)))))
   (remove (set (read-string (slurp "data/in/stop-word-num.txt"))))
   (remove (set (read-string (slurp "data/in/stop-word-kana.txt"))))
   (remove (set (read-string (slurp "data/in/stop-word1w.txt"))))))

;;word-dicの数
(def count-word-dic (count word-dic-raw3))
count-word-dic

;;word-dicでzipmapデータをつくる
(def word-dic3
  (zipmap
   (range (count word-dic-raw3))
   word-dic-raw3))

;;n番目の文章のm番目の単語がマッチするcategory No.を抽出する
(defn pickup-category-num [n m]
  (->> word-dic3
       (filter #(= (second %) (nth (meisi-token (nth data-reads n)) m)))
       first
       first))

;;それぞのn番目の文章の中全単語に適用(カテゴリベクトルの作成)
(defn category-vec [n]
  (map #(pickup-category-num n %)
       (range (count (meisi-token (nth data-reads n))))))


;;category-vectorの変形(カテゴリベクトルの頻度を抽出したmap)
(defn frequencies-category-vec [n]
  (frequencies (category-vec n)))


;;one-hot-vector作成のためのrawデータ全部でm個ある単語の中でn番目を1にする
(defn one-hot-vec [n m]
  (concat
   (vec (take (- n 1) (repeat 0)))
   [1]
   (vec (take (- m n) (repeat 0)))))

;;行列形態のone-hot-vector作成(n番目の文章でone-hot-vectorをつくる)
(defn doc-one-hot-vec [n]
  (vec (map #(vec (one-hot-vec % count-word-dic))(category-vec n))))


;;同じone-hot-vectorカウントして行列からベクトルにする(one-hot-vecを潰したもの)
;;ベクトルの足し算
(defn sum-doc-one-hot-vec [n]
  (loop [x (rest (doc-one-hot-vec n))
         y (first (doc-one-hot-vec n))]
    (if (empty? x)
      y
      (recur (rest x)(map + y (first x))))))