I want to be able to write the following functions for jdbc:
(def db {:classname "com.mysql.jdbc.Driver"
:subprotocol "mysql"
:subname "//127.0.0.1:3306/santacsv"
:server "//127.0.0.1:3306"
:schema "santa"
:user "root"
:password "root"})
(defn has-schema? [db & [schema]])
(defn create-schema [db & [schema]])
(defn drop-schema [db & [schema]])
I'm not sure how to go about it. Any help would be appreciated
At a minimum, you could use clojure.java.jdbc and write the corresponding SQL to execute!.
Alternatively, I would consider using a dedicated migrations library/framework. flyway is java-based and has a java-api which you could call with interop.
Finally, there are a few clojure-based migrations libraries like lobos, but I can't speak to their maturity, so your mileage may very.
I implemented my own here:
https://github.com/zcaudate/manas/blob/master/src/manas/core.clj
(defmacro run-statement [env database body catch-body & [statement]]
`(do (Class/forName (:classname ~env))
(let [~'conn (atom nil)]
(try
(reset! ~'conn (DriverManager/getConnection
(str "jdbc:" (:subprotocol ~env) ":"
(:server ~env) "/" (or ~database (:database ~env)))
(:user ~env)
(:password ~env)))
~#(if statement
[(list 'let '[stmt (.createStatement #conn)]
(list '.executeUpdate 'stmt statement))])
~#body
(catch SQLException ~'e
~#catch-body)
(finally
(if-not (nil? (deref ~'conn))
(.close (deref ~'conn))))))))
(defn has-connection? [env & [url]]
(let [env (if url (assoc env :server url) env)]
(run-statement env "" (true) (false))))
(defn all-databases [env]
(run-statement env ""
((let [rs (.getCatalogs (.getMetaData #conn))]
(loop [rs rs acc []]
(if (.next rs)
(recur rs (conj acc (.getString rs "TABLE_CAT")))
acc))))
()))
(defn has-database? [env & [schema]]
(run-statement env schema (true) (false)))
(defn create-database [env & [schema]]
(run-statement env ""
(true)
(false)
(str "create database " (or schema (:database env)))))
(defn drop-database [env & [schema]]
(run-statement env ""
(true)
(false)
(str "drop database " (or schema (:database env)))))
Related
How to execute the following query for thousand rows as a single batch call using a prepared statement under the hood?
(clojure.java.jdbc/execute! db ["UPDATE person SET zip = ? WHERE zip = ?" 94540 94546])
Does clojure/jdbc has an appropriate function or something else for that?
Found the answer. Applicable function is clojure.java.jdbc/db-do-prepared with enabled :multi? key.
(clojure.java.jdbc/db-do-prepared db
["UPDATE person SET zip = ? WHERE zip = ?"
[94540 94546]
[94541 94547]
...
]
{:multi? true})
Here is the way I did it for one of my projects:
(require '[clojure.java.jdbc :as sql])
(defn- db-do-cmd
[command]
(sql/db-do-commands #db-url command))
(defn- create-update-phone-sql
[{:keys [fname lname dob phone]}]
(let [where-init (format "UPDATE person SET phone = %s WHERE " (escape-quote-and-null phone))
where-rest (apply str (interpose " AND " [(str "person.dob" (escape-quote-and-null-for-where dob))
(str "person.fname" (escape-quote-and-null-for-where fname))
(str "person.lname" (escape-quote-and-null-for-where lname))]))]
(str where-init where-rest)))
(defn batch-update!
[coll]
(db-do-cmd (map create-update-phone-sql coll)))
This code works, printing the rows in the given table:
(defn count-extreferences-subset [config]
(let [emr-dbs (:emr-databases config)]
(println "Counting external references: " emr-dbs)
(jdbc/with-db-connection [dbconn (:db-spec (first emr-dbs))]
(let [q "SELECT * FROM LOCREG"
rs (jdbc/query dbconn [q])]
(dorun (map println rs))))))
According to the documentation in clojure.jdbc, this should also work, but should print the rows as the result set is realized (preventing memory overflow for large result sets):
(defn count-extreferences-subset [config]
(let [emr-dbs (:emr-databases config)]
(println "Counting external references: " emr-dbs)
(jdbc/with-db-connection [dbconn (:db-spec (first emr-dbs))]
(let [q "SELECT * FROM LOCREG"
_ (jdbc/query dbconn [q] {:row-fn println})]))))
However this fails at run-time with the following exception:
java.lang.IllegalArgumentException: No value supplied for key: {:row-fn #object[clojure.core$println 0x46ed7a0e "clojure.core$println#46ed7a0e"]}
Any idea why the use of the :row-fn option is failing?
I believe the curly braces are the problem. Your code should follow the following pattern:
(jdbc/query db-spec
["select name, cost from fruit where cost = 12"]
:row-fn add-tax)
You can fine more information in The Clojure Cookbook. I highly recommend buying a copy!
I'm trying to read millions of rows from a database and write to a text file.
This is a continuation of my question database dump to text file with side effects
My problem now seems to be that the logging doesn't happen until the program completes. Another indicator that i'm not processing lazily is that the text file isn't written at all until the program finishes.
Based on an IRC tip it seems my issue is likely having to do with :result-set-fnand defaulting to doall in the clojure.java.jdbc/query area of the code.
I have tried to replace this with a for function but still discover that memory consumption is high as it pulls the entire result set into memory.
How can i have a :result-set-fn that doesn't pull everything in like doall? How can I progressively write the log file as the program is running, rather then dump everything once the -main execution is finished?
(let [
db-spec local-postgres
sql "select * from public.f_5500_sf "
log-report-interval 1000
fetch-size 100
field-delim "\t"
row-delim "\n"
db-connection (doto ( j/get-connection db-spec) (.setAutoCommit false))
statement (j/prepare-statement db-connection sql :fetch-size fetch-size )
joiner (fn [v] (str (join field-delim v ) row-delim ) )
start (System/currentTimeMillis)
rate-calc (fn [r] (float (/ r (/ ( - (System/currentTimeMillis) start) 100))))
row-count (atom 0)
result-set-fn (fn [rs] (lazy-seq rs))
lazy-results (rest (j/query db-connection [statement] :as-arrays? true :row-fn joiner :result-set-fn result-set-fn))
]; }}}
(.setAutoCommit db-connection false)
(info "Started dbdump session...")
(with-open [^java.io.Writer wrtr (io/writer "output.txt")]
(info "Running query...")
(doseq [row lazy-results]
(.write wrtr row)
))
(info (format "Completed write with %d rows" #row-count))
)
I took the recent fixes for clojure.java.jdbc by putting [org.clojure/java.jdbc "0.3.0-beta1"] in my project.clj dependencies listing. This one enhances/corrects the :as-arrays? true functionality of clojure.java.jdbc/query described here.
I think this helped somewhat however I may still have been able to override the :result-set-fn to vec.
The core issue was resolved by tucking all row logic into :row-fn. The initial OutOfMemory problems had to do with iterating through j/query result sets rather than defining the specific :row-fn.
New (working) code is below:
(defn -main []
(let [; {{{
db-spec local-postgres
source-sql "select * from public.f_5500 "
log-report-interval 1000
fetch-size 1000
row-count (atom 0)
field-delim "\u0001" ; unlikely to be in source feed,
; although i should still check in
; replace-newline below (for when "\t"
; is used especially)
row-delim "\n" ; unless fixed-width, target doesn't
; support non-printable chars for recDelim like
db-connection (doto ( j/get-connection db-spec) (.setAutoCommit false))
statement (j/prepare-statement db-connection source-sql :fetch-size fetch-size :concurrency :read-only)
start (System/currentTimeMillis)
rate-calc (fn [r] (float (/ r (/ ( - (System/currentTimeMillis) start) 100))))
replace-newline (fn [s] (if (string? s) (clojure.string/replace s #"\n" " ") s))
row-fn (fn [v]
(swap! row-count inc)
(when (zero? (mod #row-count log-report-interval))
(info (format "wrote %d rows" #row-count))
(info (format "\trows/s %.2f" (rate-calc #row-count)))
(info (format "\tPercent Mem used %s " (memory-percent-used))))
(str (join field-delim (doall (map #(replace-newline %) v))) row-delim ))
]; }}}
(info "Started database table dump session...")
(with-open [^java.io.Writer wrtr (io/writer "./sql/output.txt")]
(j/query db-connection [statement] :as-arrays? true :row-fn
#(.write wrtr (row-fn %))))
(info (format "\t\t\tCompleted with %d rows" #row-count))
(info (format "\t\t\tCompleted in %s seconds" (float (/ (- (System/currentTimeMillis) start) 1000))))
(info (format "\t\t\tAverage rows/s %.2f" (rate-calc #row-count)))
nil)
)
Other things i experimented (with limited success) involved the timbre logging and turning off stardard out; i wondered if with using a REPL it might cache the results before displaying back to my editor (vim fireplace) and i wasn't sure if that was utilizing a lot of the memory.
Also, I added the logging parts around memory free with (.freeMemory (java.lang.Runtime/getRuntime)). I wasn't as familiar with VisualVM and pinpointing exactly where my issue was.
I am happy with how it works now, thanks everyone for your help.
You can use prepare-statement with the :fetch-size option. Otherwise, the query itself is eager despite the results being delivered in a lazy sequence.
prepare-statement requires a connection object, so you'll need to explicitly create one. Here's an example of how your usage might look:
(let [db-spec local-postgres
sql "select * from big_table limit 500000 "
fetch-size 10000 ;; or whatever's appropriate
cnxn (doto (j/get-connection db-spec)
(.setAutoCommit false))
stmt (j/prepare-statement cnxn sql :fetch-size fetch-size)
results (rest (j/query cnxn [stmt]))]
;; ...
)
Another option
Since the problem seems to be with query, try with-query-results. It's considered deprecated but is still there and works. Here's an example usage:
(let [db-spec local-postgres
sql "select * from big_table limit 500000 "
fetch-size 100 ;; or whatever's appropriate
cnxn (doto (j/get-connection db-spec)
(.setAutoCommit false))
stmt (j/prepare-statement cnxn sql :fetch-size fetch-size)]
(j/with-query-results results [stmt] ;; binds the results to `results`
(doseq [row results]
;;
)))
I've have found a better solution: you need to declare a cursor and fetch chunks of data from it in a transaction. Example:
(db/with-tx
(db/execute! "declare cur cursor for select * from huge_table")
(loop []
(when-let [rows (-> "fetch 10 from cur" db/query not-empty)]
(doseq [row rows]
(process-a-row row))
(recur))))
Here, db/with-tx, db/execute! and db/query are my own shortcuts declared in db namespace:
(def ^:dynamic
*db* {:dbtype "postgresql"
:connection-uri <some db url>)})
(defn query [& args]
(apply jdbc/query *db* args))
(defn execute! [& args]
(apply jdbc/execute! *db* args))
(defmacro with-tx
"Runs a series of queries into transaction."
[& body]
`(jdbc/with-db-transaction [tx# *db*]
(binding [*db* tx#]
~#body)))
why do i get exception on (redirect/get) in this program
#lang web-server
(require web-server/formlets web-server/page) (struct app (nm) #:mutable)
(define (start req) (render-main-page req))
this function is to be used by most pages and generates comlete page xexpr by calling each given piece of page generator functions, each of which may embed their urls
(define (render-page embed/url a-title blocks)
(response/xexpr `(html (head (title ,a-title)
(body ,#(map (lambda (block) (block embed/url)) blocks))))))
this is piece of first page generator function
(define (render-action-panel embed/url action)
`(a ([href ,(embed/url action)]) "New"))
this is first page
(define/page (render-main-page)
(local [(define (new-handler req) (render-app-page req (app "new value")))
(define (panel-block embed/url) (render-action-panel embed/url new-handler))]
(render-page embed/url "Title" (list panel-block))))
this is piece of second page generator function (represents form)
(define (add-app-formlet an-app) (formlet (#%# ,{=> input-string nm}) nm))
(define (render-app-form embed/url an-app save-handler)
`(div (form ([action ,(embed/url save-handler)][method "POST"])
(span ,#(formlet-display (add-app-formlet an-app) ))
(span (input ([type "submit"][value "Save"]))))));)
the second form,
save handler throws exception when trying do post-redirect-get
(define/page (render-app-page an-app)
(local [(define (save-handler req)
(render-app-page
(redirect/get)
(set-app-nm! an-app (formlet-process (add-app-formlet an-app) req))))
(define (form-block embed/url)
(render-app-form embed/url an-app save-handler ))
]
(render-page embed/url "Title - form: " (list form-block))))
(require web-server/servlet-env)
(serve/servlet start)
Which redirect/get are you using?
The one from web-server/lang/servlet (which should be used with #lang web-server) is different than the one from web-server/servlet (which should be used with #lang racket (and friends))
This error message means that you are using the one from web-server/servlet.
FWIW, web-server/page cannot be used with #lang web-server because it is just a simple macro over the send/suspend/dispatch from web-server/servlet.
Given a set of delimited data in the following format:
1|Star Wars: Episode IV - A New Hope|1977|Action,Sci-Fi|George Lucas
2|Titanic|1997|Drama,History,Romance|James Cameron
In elisp, how can I generate sql insert statements in this format?
insert into table
values(1,"Star Wars: Episode IV - A New Hope",1977","Action,Sci-Fi","George Lucas",0);
insert into table
values(2,"Titanic",1997,"Drama,History,Romance","James Cameron",0);
To simplify the problem, let's allow for a parameter to tell which
columns are text or numeric. (e.g. 0,1,0,1,1)
Here's how I would do it in Perl.
my #ctypes=qw/0 1 0 1 1/;
while(<>) {
chop;
#F=split('\|', $_);
print "insert into table values(";
foreach my $col (#F) {
my $type=shift(#ctypes);
print ($type == 1 ? '"'.$col.'"' : $col);
print ",";
}
print "0);\n";
}
To insert the data at the end of a buffer:
(require 'cl)
(defun* insert-statements (rows ctypes &key (table-name "table") (delimiter "|"))
(let* ((values-template
(mapconcat '(lambda (type) (if (= type 1) "\"%s\"" "%s")) ctypes ","))
(template (format "insert into %s values(%s);\n" table-name values-template)))
(mapcar '(lambda (row) (insert (apply 'format (cons template (split-string row delimiter)))))
rows)))
(let ((data "1|Star Wars: Episode IV - A New Hope|1977|Action,Sci-Fi|George Lucas
2|Titanic|1997|Drama,History,Romance|James Cameron")
(ctypes '(0 1 0 1 1)))
(save-excursion
(goto-char (point-max))
(insert-statements (split-string data "\n") ctypes)))
This solution may not scale well compared with the Perl solution.
Alternatively, we could insert the values into a database using SqlMode:
(require 'cl)
(defun* insert-statements (rows ctypes &key (table-name "table") (delimiter "|"))
(let* ((values-template
(mapconcat '(lambda (type) (if (= type 1) "\"%s\"" "%s")) ctypes ","))
(template (format "insert into %s values(%s);" table-name values-template)))
(mapcar '(lambda (row) (sql-send-string
(apply 'format (cons template (split-string row delimiter)))) )
rows)))
(sql-sqlite)
(sql-send-string "create table test (id, title, yr, genre, director);")
(let ((data "1|Star Wars: Episode IV - A New Hope|1977|Action,Sci-Fi|George Lucas
2|Titanic|1997|Drama,History,Romance|James Cameron")
(ctypes '(0 1 0 1 1)))
(insert-statements (split-string data "\n") ctypes :table-name 'test))
(sql-send-string "select title from test;")
(sql-send-string "drop table test;")
you can write a simple program to go through each line and look for delimiter | and generate insert statements. if you are familiar with using mysql you can import the data into mysql table using "load data" command. you will have to have the data in file though