Merge pull request #93 from tombarys/apriori

Update several things, corrected typos and ugly things, added simple core correlation code and credits

Author: timothypratley

src/data_analysis/book_sales_analysis/about_apriori.clj

@@ -21,6 +21,7 @@
             [data-analysis.book-sales-analysis.core-helpers-v2 :as helpers]
             [data-analysis.book-sales-analysis.market-basket-analysis-v2 :as mba]))
 
+
 ;; # From Correlations to Recommendations
 ;; ## A Publisher's Journey into Data-Driven Book Sales
 
@@ -95,6 +96,31 @@
 
 ;; My first instinct was to calculate correlations between all books. A correlation tells you how often two books appear together compared to what you'd expect by chance. When I visualized this as a heatmap, with books ordered chronologically, something fascinating emerged:
 
+;; The core of the correlation calculation is this function (thank you, [@generateme](https://github.com/generateme) for huge [help](https://clojurians.zulipchat.com/#narrow/channel/151924-data-science/topic/Correlation.20matrix.20best.20practice.3F/with/530339272)):
+
+^:kindly/hide-code
+(kind/code
+ "; ...
+  
+(defn corr-a-x-b [ds]
+  (let
+   [columns (tc/column-names ds)
+    clean-ds  (tc/drop-columns ds [:zakaznik]))]
+    (-> (zipmap columns (stats/correlation-matrix (tc/columns clean-ds)))
+        tc/dataset
+        (tc/add-column :book columns))))
+  
+; ...")
+
+;; ...which after chronological sorting (see article's repo) enters this plotly element:
+
+^:kindly/hide-code
+(def example-corr-matrix-calculation 
+  (->  data/anonymized-shareable-ds
+       helpers/onehot-encode-by-customers
+       (helpers/corr-matrix data/books-meta)))
+
+
 (kind/plotly
  {:data [{:type "heatmap"
           :z (tc/columns data/corr-matrix-precalculated)
@@ -114,7 +140,7 @@
                           :font {:size 12 :color "black"}
                           :bgcolor "rgba(255, 255, 200, 0.9)" :bordercolor "yellow" :borderwidth 2}]}})
 
-;; The bright red square in the upper-left corner revealed that **recently published books have much stronger co-purchase patterns** than older titles. This made intuitive sense—customers discovering our catalog tend to buy multiple new releases together.
+;; The bright red square in the lower-left corner revealed that **recently published books have much stronger co-purchase patterns** than older titles. This made intuitive sense—customers discovering our catalog tend to buy multiple new releases together.
 
 ;; ## A Surprising Discovery: Czech vs. Foreign Authors
 
@@ -234,7 +260,7 @@ scatter-plot
 ;; **Lift** measures whether this happens more than random chance:
 
 ;; $$ 
-;; "\text{Lift}(A \rightarrow B) = \dfrac{\text{Confidence}(A \rightarrow B)}{\text{Support}(\{B\})}
+;; \text{Lift}(A \rightarrow B) = \dfrac{\text{Confidence}(A \rightarrow B)}{\text{Support}(\{B\})}
 ;; $$ 
 
 ;; A lift greater than 1 indicates positive association—the items are purchased together more often than if they were independent. A lift of 2.3 means the combination is 2.3 times more likely than chance.
@@ -255,7 +281,7 @@ scatter-plot
  (tc/head (:rules-grouped quick-formatted) 15)
  {:element/max-height "500px"})
 
-;; Reading these rules is straightforward. For example, if a customer buys Yuval Noah Harari's "Sapiens," there's a 72% chance they'll also purchase "Nexus" (another Harari title), and this combination is nearly twice as likely as random chance (lift = 1.99).
+;; Reading these rules is straightforward. For example, if a customer buys "Zamilujte se do angličtiny", there's a 37% chance they'll also purchase "365 anglických frází a výrazů" (another author's title), and this combination is 6 times as likely as random chance (lift = 6,12).
 
 ;; ## Visualizing the Network
 
@@ -344,6 +370,15 @@ scatter-plot
 ;; - Full presentation code: *to be published*
 ;; - SciCloj community: [scicloj.github.io](https://scicloj.github.io)
 
+;; ## Credits
+
+;; I would like to thank [Daniel Slutsky](https://github.com/daslu) for inviting me to participate in the conference and encouraging me all the time not to give up. I think he is doing an amazing job for this great community. 
+;; I would also like to thank [Timothy Pratley](https://github.com/timothypratley) for his support during the publication process. 
+;; Extra thanks to my friends at [Not Null Makers](https://notnullmakers.com/) for making me a better Clojure person :).
+ 
+
+
+
 ;; ---
 
 ;; *This article is based on a presentation at Macroexpand conference, October 2025.*

src/data_analysis/book_sales_analysis/books_meta.nippy

@@ -112,8 +112,8 @@
                    (map #(str/replace % #"\+" ""))
                    (map #(str/trim %))
                    (map sanitize-str)
-                   (map #(str/replace % #"\-\-.+$" "")) 
-                   (map #(str/replace % #"\-+$" "")) 
+                   (map #(str/replace % #"\-\-.+$" ""))
+                   (map #(str/replace % #"\-+$" ""))
                    (map #(str/replace % #"^3" "k3"))
                    (map #(str/replace % #"^5" "k5"))
                    (remove (fn [item] (some (fn [substr] (str/includes? (name item) substr))
@@ -125,6 +125,28 @@
 
 ;; ### Metadata Enriching and Convenience Functions
 
+(def end-time
+  (jt/local-date 2025 10 1))
+
+(defn months-between "Calculate how many months a product has been on market"
+  [start-date end-date]
+  (let [days (if (and start-date end-date)
+               (jt/time-between start-date end-date :days)
+               0)]
+    (long (Math/round (/ days 30.4375)))))
+
+(defn months-on-market
+  "Months `book` is on a market. Zero if not at all."
+  [books-ds book end-date]
+  (let [date (try
+               (-> books-ds
+                   (tc/select-columns [:titul :datum-zahajeni-prodeje])
+                   (tc/select-rows #(str/starts-with? (name (:titul %)) (name book)))
+                   (tc/get-entry :datum-zahajeni-prodeje 0))
+               (catch Exception e nil))
+        month (if (nil? date) 0 (months-between date end-date))]
+    month))
+
 (defn czech-author? [book-title]
   (let [czech-books #{:k30-hodin
                       :k365-anglickych-cool-fraz-a-vyrazov
@@ -159,7 +181,7 @@
 
 ;; ### One-Hot Encoding Functions
 
-   
+
 (defn onehot-encode-by-customers ;; FIXME needs refactor and simplification :)
   "One-hot encode dataset aggregated by customer. 
    Each customer gets one row with 0/1 values for each book they bought.
@@ -279,25 +301,23 @@
         tc/dataset
         (tc/add-column :book columns))))
 
-(defn corr-3-col
-  "Creates a correlation matrix with two columns of books \n
-  => _unnamed [4 3]: \n
-      | :book-0 | :book-1 | :correlation |
-      |---------|---------|-------------:|
-      |      :a |      :a |   1.00000000 |
-      |      :a |      :b |  -0.12121831 |
-      |      :b |      :a |  -0.12121831 |
-      |      :b |      :b |   1.00000000 | \n
-  - `flatten` is used here to make a linear sequence of numbers which should match corresponding variable names. \n
-  -  Since we make pairs of names `((for...[a b])` creates a cartesian product) we need to separate these to individual columns, tc/seperate-column does the trick, refer: https://scicloj.github.io/tablecloth/#separate"
-  [ds]
-  (let [names (tc/column-names ds)
-        mat (flatten (stats/correlation-matrix (tc/columns ds)))]
-    (-> (tc/dataset {:book (for [a names b names] [a b])
-                     :correlation mat})
-        (tc/separate-column :book)
-        (tc/rename-columns  {":book-0" :titul-knihy
-                             ":book-1" :book-1}))))
+(defn corr-matrix
+  "Creates a correlation matrix with books sorted by publication date (chronological order) \n
+   `books-onehot` – one-hot encoded dataset"
+  [books-onehot books-meta]
+  (-> (corr-a-x-b (-> books-onehot
+                      (tc/reorder-columns
+                       (sort-by #(months-on-market books-meta % end-time)
+                                (tc/column-names books-onehot)))
+                      (tc/drop-columns [:zakaznik])))
+      (tc/reorder-columns
+       (sort-by #(months-on-market books-meta % end-time)
+                (tc/column-names books-onehot)))
+      (tc/add-column :sort-col
+                     (fn [ds] (map #(months-on-market books-meta % end-time)
+                                   (tc/column ds :book))))
+      (tc/order-by :sort-col)
+      (tc/drop-columns :sort-col)))
 
 
 ;; ### Export helper functions from other namespaces for convenience

src/data_analysis/book_sales_analysis/core_helpers_v2.clj

@@ -7,16 +7,6 @@
 
 ;; ### Main Orders Data (WooCommerce exports)
 
-#_(def anonymized-presentation-ds
-  "Anonymized customers full dataset of all orders - for presentation purposes
-   ❗ NOT FOR SHARING"
-  (tc/dataset
-   (helpers/merge-csvs
-    ["data/anonymized-customers-only-presentation-v2.csv"]
-    {:header? true
-     :separator ","
-     :key-fn #(keyword (helpers/sanitize-str %))})))
-
 (def anonymized-shareable-ds
   "Fully anonymized slice of db - for sharing purposes
    ✅ SAFE TO SHARE"
@@ -29,13 +19,15 @@
 
 ;; ### Book Metadata
 
+(def books-meta
+  (tc/dataset "src/data_analysis/book_sales_analysis/books_meta.nippy"))
+
 ;; ## Quick Access
 ;; Most commonly used datasets with short aliases
 
 (def orders-share anonymized-shareable-ds)
-#_(def orders-slides anonymized-presentation-ds)
 
-(def corr-matrix-precalculated 
+(def corr-matrix-precalculated
   (tc/dataset "src/data_analysis/book_sales_analysis/corr-matrix.nippy"))
 
 corr-matrix-precalculated

src/data_analysis/book_sales_analysis/data_sources_v2.clj

@@ -12,7 +12,6 @@
             [clojure.math.combinatorics :as combo]
             [clojure.string :as str]
             [clojure.set]
-            [data-analysis.book-sales-analysis.data-sources-v2 :as data]
             [data-analysis.book-sales-analysis.core-helpers-v2 :as helpers]))
 
 ;; # Market Basket Analysis