Add a new config to scrap description from babelio

This commit is contained in:
2026-03-24 17:02:43 +01:00
parent c4390742b3
commit 843c5b5dbc
12 changed files with 347 additions and 46 deletions

10
go.mod
View File

@@ -3,6 +3,7 @@ module git.artlef.fr/bibliomane
go 1.26 go 1.26
require ( require (
github.com/PuerkitoBio/goquery v1.12.0
github.com/alecthomas/kong v1.14.0 github.com/alecthomas/kong v1.14.0
github.com/alecthomas/kong-toml v0.4.0 github.com/alecthomas/kong-toml v0.4.0
github.com/gin-gonic/gin v1.11.0 github.com/gin-gonic/gin v1.11.0
@@ -11,13 +12,14 @@ require (
github.com/nicksnyder/go-i18n/v2 v2.6.0 github.com/nicksnyder/go-i18n/v2 v2.6.0
github.com/pelletier/go-toml v1.9.5 github.com/pelletier/go-toml v1.9.5
github.com/stretchr/testify v1.11.1 github.com/stretchr/testify v1.11.1
golang.org/x/crypto v0.48.0 golang.org/x/crypto v0.49.0
golang.org/x/text v0.34.0 golang.org/x/text v0.35.0
gorm.io/driver/sqlite v1.6.0 gorm.io/driver/sqlite v1.6.0
gorm.io/gorm v1.31.0 gorm.io/gorm v1.31.0
) )
require ( require (
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/gopkg v0.1.3 // indirect
github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic v1.15.0 // indirect
github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect
@@ -47,8 +49,8 @@ require (
github.com/ugorji/go/codec v1.3.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect
go.uber.org/mock v0.6.0 // indirect go.uber.org/mock v0.6.0 // indirect
golang.org/x/arch v0.24.0 // indirect golang.org/x/arch v0.24.0 // indirect
golang.org/x/net v0.50.0 // indirect golang.org/x/net v0.52.0 // indirect
golang.org/x/sys v0.41.0 // indirect golang.org/x/sys v0.42.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
) )

85
go.sum
View File

@@ -1,5 +1,7 @@
github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo=
github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/kong v1.14.0 h1:gFgEUZWu2ZmZ+UhyZ1bDhuutbKN1nTtJTwh19Wsn21s= github.com/alecthomas/kong v1.14.0 h1:gFgEUZWu2ZmZ+UhyZ1bDhuutbKN1nTtJTwh19Wsn21s=
@@ -8,6 +10,8 @@ github.com/alecthomas/kong-toml v0.4.0 h1:sSK/HHi2M5jqSXYTxmuxkdZcJ+ip9jhYvwcjDG
github.com/alecthomas/kong-toml v0.4.0/go.mod h1:hRVV9iGmqYsFqs17jFQgqhkjYIxiklbfy95xJ3nlpKI= github.com/alecthomas/kong-toml v0.4.0/go.mod h1:hRVV9iGmqYsFqs17jFQgqhkjYIxiklbfy95xJ3nlpKI=
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE=
@@ -40,6 +44,7 @@ github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -97,19 +102,83 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY=
github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y=
golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A=
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@@ -10,3 +10,5 @@ image-folder-path = "/tmp"
# The port to listen on for the server. # The port to listen on for the server.
port = "8080" port = "8080"
book-description-from-babelio = true

View File

@@ -24,6 +24,10 @@ func TestPostImportBookHandler_Ok(t *testing.T) {
assert.Equal(t, "Emily Brontë", book.Author) assert.Equal(t, "Emily Brontë", book.Author)
assert.Equal(t, "isbn:9782253004752", book.InventaireId) assert.Equal(t, "isbn:9782253004752", book.InventaireId)
assert.Equal(t, "/static/bookcover/44abbcbdc1092212c2bae66f5165019dac1e2a7b.webp", book.CoverPath) assert.Equal(t, "/static/bookcover/44abbcbdc1092212c2bae66f5165019dac1e2a7b.webp", book.CoverPath)
expectedDesc := `Roman unique, à la croisée du fantastique et du romantisme, ce texte inclassable bouleverse les codes du XIXe siècle par sa violence émotionnelle, sa narration fragmentée et ses personnages à fleur de peau.
Sur les landes battues par les vents, à l'ombre des murs de Hurlevent, se joue une tragédie d'amour et de vengeance entre Catherine et Heathcliff - deux âmes tourmentées, liées par une passion aussi absolue que destructrice.
Sublimée par l'univers graphique intense d'Isabella Mazzanti, cette édition s'impose comme un objet littéraire à part, mêlant innovations narratives et force d'évocation. Les images semblent vibrer d'un souffle secret, comme si le vent y faisait surgir, en silence, le tumulte des passions.`
assert.Equal(t, expectedDesc, book.Summary)
} }
func TestPostImportBookHandler_OkAuthorKey(t *testing.T) { func TestPostImportBookHandler_OkAuthorKey(t *testing.T) {
@@ -33,6 +37,9 @@ func TestPostImportBookHandler_OkAuthorKey(t *testing.T) {
assert.Equal(t, "Philip K. Dick", book.Author) assert.Equal(t, "Philip K. Dick", book.Author)
assert.Equal(t, "isbn:9782290033630", book.InventaireId) assert.Equal(t, "isbn:9782290033630", book.InventaireId)
assert.Equal(t, "/static/bookcover/1d1493159d031224a42b37c4417fcbb8c76b00bd.webp", book.CoverPath) assert.Equal(t, "/static/bookcover/1d1493159d031224a42b37c4417fcbb8c76b00bd.webp", book.CoverPath)
expectedDesc := `Les bombes étaient finalement tombées. Malgré l'équilibre de la terreur, un jour un homme avait été assez fou pour appuyer sur le bouton. Cependant, dans ce coin perdu de Californie, la vie continuait. Pour Bonny Keller, toujours perturbée malgré six ans d'analyse ; pour Bruno Bluthgeld, l'un des responsables de la grande catastrophe ; pour Hoppy, le phocomèle, l'ancien bébé thalidomide doté de pouvoirs supranormaux... Elle continuait aussi pour Walt Dangerfield, l'astronaute expédié sur mars, mais dont la cabine s'était satellisée autour de la terre. Là, à l'abri des radiations, il s'était transformé en une sorte de super disc-jockey dont l'écoute était devenue une drogue pour tous les survivants...
Philip K. Dick (1928-1982). A travers une œuvre imposante, il ne cessera de traiter ses thèmes de prédilection : la juxtaposition de deux niveaux de réalité — l'un "objectivement" déterminé, l'autre n'étant qu'un monde d'apparences — et la poranoïa qu'impliquent ces manipulations de la réalité dont personne ne tonnait jamais le degré exact de "virtualité".`
assert.Equal(t, expectedDesc, book.Summary)
} }
func TestPostImportBookHandler_NoOLID(t *testing.T) { func TestPostImportBookHandler_NoOLID(t *testing.T) {

128
internal/babelio/babelio.go Normal file
View File

@@ -0,0 +1,128 @@
package babelio
import (
"errors"
"fmt"
"io"
"net/http"
"strings"
"git.artlef.fr/bibliomane/internal/callapiutils"
"git.artlef.fr/bibliomane/internal/myvalidator"
"github.com/PuerkitoBio/goquery"
"golang.org/x/text/encoding/charmap"
)
type babelioSearchArg struct {
Term string `json:"term"`
}
type babelioSearchResult struct {
//only parsing the url
Url string `json:"url"`
}
func GetDescriptionFromISBN(baseUrl string, isbn string) (string, error) {
url, err := searchPageIsbn(baseUrl, isbn)
if err != nil {
return "", err
}
//we either find the full summary, or we have to make another call to get it.
fullSummary, payloadToQuery, err := parseBookPage(baseUrl, url)
if err != nil {
return "", err
}
if fullSummary != "" {
return decodeAndCleanText(strings.NewReader(fullSummary)), err
} else if payloadToQuery != "" {
return queryDescription(baseUrl, payloadToQuery)
} else {
return "", nil
}
}
func searchPageIsbn(baseUrl, isbn string) (string, error) {
searchUrl, err := callapiutils.ComputeUrl(baseUrl, "aj_recherche.php")
if err != nil {
return "", err
}
term := babelioSearchArg{Term: isbn}
var searchResults []babelioSearchResult
callapiutils.FetchAndParseResultFromPost(searchUrl, &term, &searchResults)
if len(searchResults) == 0 {
return "", myvalidator.TranslatedError{Err: errors.New("ISBNNotFoundBabelio")}
}
return searchResults[0].Url, nil
}
func parseBookPage(baseUrl, bookUrl string) (string, string, error) {
url, err := callapiutils.ComputeUrl(baseUrl, bookUrl)
if err != nil {
return "", "", err
}
resp, err := http.Get(url.String())
if err != nil {
return "", "", err
}
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromReader(resp.Body)
//we either find the full summary, or we have to make another call to get it.
fullsummary := ""
jsToParse := ""
doc.Find(".livre_resume").Each(func(i int, s *goquery.Selection) {
onclick, ok := s.Find("a").Attr("onclick")
if ok {
jsToParse = onclick
} else {
fullsummary = s.Text()
}
})
if fullsummary != "" {
return fullsummary, "", nil
}
typeStr, idObj, err := extractNumbersFromExpression(jsToParse)
if err != nil {
return "", "", err
}
return "", fmt.Sprintf("type=%s&id_obj=%s", typeStr, idObj), nil
}
func extractNumbersFromExpression(jsToParse string) (string, string, error) {
splitted := strings.Split(jsToParse, ",")
if len(splitted) < 3 {
return "", "", myvalidator.TranslatedError{Err: errors.New("BabelioParseError")}
}
if len(splitted[2]) < 3 {
return "", "", myvalidator.TranslatedError{Err: errors.New("BabelioParseError")}
}
return splitted[1], splitted[2][:len(splitted[2])-2], nil
}
func queryDescription(baseUrl string, payloadToQuery string) (string, error) {
url, err := callapiutils.ComputeUrl(baseUrl, "aj_voir_plus_a.php")
if err != nil {
return "", err
}
resp, err := http.Post(url.String(),
"application/x-www-form-urlencoded; charset=UTF-8",
strings.NewReader(payloadToQuery))
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", myvalidator.TranslatedError{Err: fmt.Errorf("BabelioFetchDescError")}
}
return decodeAndCleanText(resp.Body), nil
}
func decodeAndCleanText(reader io.Reader) string {
tr := charmap.Windows1252.NewDecoder().Reader(reader)
var decodedString strings.Builder
io.Copy(&decodedString, tr)
return strings.TrimSpace(strings.ReplaceAll(decodedString.String(), "<br>", "\n"))
}

View File

@@ -0,0 +1,30 @@
package babelio
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetDescriptionFromISBN_Philip(t *testing.T) {
desc, err := GetDescriptionFromISBN("https://www.babelio.com", "9782290033630")
expectedDesc := `Les bombes étaient finalement tombées. Malgré l'équilibre de la terreur, un jour un homme avait été assez fou pour appuyer sur le bouton. Cependant, dans ce coin perdu de Californie, la vie continuait. Pour Bonny Keller, toujours perturbée malgré six ans d'analyse ; pour Bruno Bluthgeld, l'un des responsables de la grande catastrophe ; pour Hoppy, le phocomèle, l'ancien bébé thalidomide doté de pouvoirs supranormaux... Elle continuait aussi pour Walt Dangerfield, l'astronaute expédié sur mars, mais dont la cabine s'était satellisée autour de la terre. Là, à l'abri des radiations, il s'était transformé en une sorte de super disc-jockey dont l'écoute était devenue une drogue pour tous les survivants...
Philip K. Dick (1928-1982). A travers une œuvre imposante, il ne cessera de traiter ses thèmes de prédilection : la juxtaposition de deux niveaux de réalité — l'un "objectivement" déterminé, l'autre n'étant qu'un monde d'apparences — et la poranoïa qu'impliquent ces manipulations de la réalité dont personne ne tonnait jamais le degré exact de "virtualité".`
if err != nil {
t.Error(err)
return
}
assert.Equal(t, expectedDesc, desc)
}
func TestGetDescriptionFromISBN_Emily(t *testing.T) {
desc, err := GetDescriptionFromISBN("https://www.babelio.com", "9782253004752")
expectedDesc := `Roman unique, à la croisée du fantastique et du romantisme, ce texte inclassable bouleverse les codes du XIXe siècle par sa violence émotionnelle, sa narration fragmentée et ses personnages à fleur de peau.
Sur les landes battues par les vents, à l'ombre des murs de Hurlevent, se joue une tragédie d'amour et de vengeance entre Catherine et Heathcliff - deux âmes tourmentées, liées par une passion aussi absolue que destructrice.
Sublimée par l'univers graphique intense d'Isabella Mazzanti, cette édition s'impose comme un objet littéraire à part, mêlant innovations narratives et force d'évocation. Les images semblent vibrer d'un souffle secret, comme si le vent y faisait surgir, en silence, le tumulte des passions.`
if err != nil {
t.Error(err)
return
}
assert.Equal(t, expectedDesc, desc)
}

View File

@@ -1,6 +1,7 @@
package callapiutils package callapiutils
import ( import (
"bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
@@ -20,9 +21,33 @@ func AddQueryParam(u *url.URL, paramName string, paramValue string) {
u.RawQuery = q.Encode() u.RawQuery = q.Encode()
} }
func FetchAndParseResultFromPost[T any, J any](u *url.URL, queryArg *J, queryResult *T) error {
payloadBuf := new(bytes.Buffer)
json.NewEncoder(payloadBuf).Encode(queryArg)
req, err := http.NewRequest("POST", u.String(), payloadBuf)
if err != nil {
return err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
log.Printf("Calling POST %s", u.String())
return parseApiQueryResult(u, req, queryResult)
}
func FetchAndParseResult[T any](u *url.URL, queryResult *T) error { func FetchAndParseResult[T any](u *url.URL, queryResult *T) error {
resp, err := DoApiQuery(u) req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return err
}
req.Header.Add("Accept", "application/json")
req.Header.Add("User-Agent", "bibliomane/0.1 (artlef@protonmail.com)")
log.Printf("Calling GET %s", u.String()) log.Printf("Calling GET %s", u.String())
return parseApiQueryResult(u, req, queryResult)
}
func parseApiQueryResult[T any](u *url.URL, req *http.Request, queryResult *T) error {
client := &http.Client{}
resp, err := client.Do(req)
if err != nil { if err != nil {
return err return err
} }
@@ -42,18 +67,8 @@ func FetchAndParseResult[T any](u *url.URL, queryResult *T) error {
if err != nil { if err != nil {
return err return err
} }
return err
}
func DoApiQuery(u *url.URL) (*http.Response, error) { return err
client := &http.Client{}
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
}
req.Header.Add("Accept", "application/json")
req.Header.Add("User-Agent", "bibliomane/0.1 (artlef@protonmail.com)")
return client.Do(req)
} }
func ComputeUrl(baseUrl string, paths ...string) (*url.URL, error) { func ComputeUrl(baseUrl string, paths ...string) (*url.URL, error) {

View File

@@ -22,17 +22,19 @@ type CLI struct {
} }
type Config struct { type Config struct {
Port string `toml:"port" short:"p" default:"8080" help:"Port to listen on for the server." comment:"Port to listen on for the server."` Port string `toml:"port" short:"p" default:"8080" help:"Port to listen on for the server." comment:"Port to listen on for the server."`
DatabaseFilePath string `toml:"database-file-path" short:"d" default:"bibliomane.db" type:"path" help:"Path to sqlite database file." comment:"Path to sqlite database file."` DatabaseFilePath string `toml:"database-file-path" short:"d" default:"bibliomane.db" type:"path" help:"Path to sqlite database file." comment:"Path to sqlite database file."`
DemoDataPath string `toml:"demo-data-path" help:"Path to the sql file to load for demo data." comment:"Path to the sql file to load for demo data."` DemoDataPath string `toml:"demo-data-path" help:"Path to the sql file to load for demo data." comment:"Path to the sql file to load for demo data."`
JWTKey string `toml:"jwt-key" help:"Key used to encrypt JWT." comment:"Key used to encrypt the generated JWT. Encoded in base64. If empty a random one will be generated on every restart."` JWTKey string `toml:"jwt-key" help:"Key used to encrypt JWT." comment:"Key used to encrypt the generated JWT. Encoded in base64. If empty a random one will be generated on every restart."`
ImageFolderPath string `toml:"image-folder-path" short:"i" default:"img" type:"path" help:"Folder where uploaded files will be stored." comment:"Folder where uploaded files will be stored."` ImageFolderPath string `toml:"image-folder-path" short:"i" default:"img" type:"path" help:"Folder where uploaded files will be stored." comment:"Folder where uploaded files will be stored."`
Limit int `toml:"limit" default:"100" help:"A single API call will return at most this number of records." comment:"A single API call will return at most this number of records."` Limit int `toml:"limit" default:"100" help:"A single API call will return at most this number of records." comment:"A single API call will return at most this number of records."`
InventaireUrl string `toml:"inventaire-url" default:"https://inventaire.io" help:"An inventaire.io instance URL." comment:"An inventaire.io instance URL."` InventaireUrl string `toml:"inventaire-url" default:"https://inventaire.io" help:"An inventaire.io instance URL." comment:"An inventaire.io instance URL."`
DisableRegistration bool `toml:"disable-registration" short:"n" default:"false" help:"Disable new account creation." comment:"Disable new account creation."` BookDescriptionFromBabelio bool `toml:"book-description-from-babelio" default:"false" help:"Activate fetching description from babelio.com." comment:"Activate fetching description from babelio.com."`
DemoMode bool `toml:"demo-mode" short:"D" default:"false" help:"Activate demo mode." comment:"Activate demo mode: anyone connecting to the instance will be logged in as a single user."` BabelioUrl string `toml:"babelio-url" default:"https://www.babelio.com" comment:"Link to babelio website."`
DemoUsername string `toml:"demo-username" default:"demo" help:"Name of the single user used for the demo." comment:"Name of the single user used for the demo."` DisableRegistration bool `toml:"disable-registration" short:"n" default:"false" help:"Disable new account creation." comment:"Disable new account creation."`
AddUser UserListAsStrings `toml:"add-user" short:"a" help:"Add users on startup following htpasswd bcrypt format." comment:"Add users on startup following htpasswd bcrypt format, example: [\"demo:$2y$10$UHR2646SZo2W.Rhna7bn5eWNLXWJZ/Sa3oLd9RlxlXs57Bwp6isOS\",\"user:$2y$10$3WYUp.VDpzJRywtrxO1s/uWfUIKpTE4yh5B1d2RCef3hvczYbEWTC\"]"` DemoMode bool `toml:"demo-mode" short:"D" default:"false" help:"Activate demo mode." comment:"Activate demo mode: anyone connecting to the instance will be logged in as a single user."`
DemoUsername string `toml:"demo-username" default:"demo" help:"Name of the single user used for the demo." comment:"Name of the single user used for the demo."`
AddUser UserListAsStrings `toml:"add-user" short:"a" help:"Add users on startup following htpasswd bcrypt format." comment:"Add users on startup following htpasswd bcrypt format, example: [\"demo:$2y$10$UHR2646SZo2W.Rhna7bn5eWNLXWJZ/Sa3oLd9RlxlXs57Bwp6isOS\",\"user:$2y$10$3WYUp.VDpzJRywtrxO1s/uWfUIKpTE4yh5B1d2RCef3hvczYbEWTC\"]"`
} }
type UserListAsStrings []string type UserListAsStrings []string
@@ -48,17 +50,19 @@ func (u UserListAsStrings) Validate() error {
func defaultConfig() CLI { func defaultConfig() CLI {
c := Config{ c := Config{
Port: "8080", Port: "8080",
DatabaseFilePath: "bibliomane.db", DatabaseFilePath: "bibliomane.db",
DemoDataPath: "", DemoDataPath: "",
JWTKey: "", JWTKey: "",
ImageFolderPath: "img", ImageFolderPath: "img",
Limit: 100, Limit: 100,
InventaireUrl: "https://inventaire.io", InventaireUrl: "https://inventaire.io",
DisableRegistration: false, BookDescriptionFromBabelio: false,
DemoMode: false, BabelioUrl: "https://www.babelio.com",
DemoUsername: "demo", DisableRegistration: false,
AddUser: []string{}, DemoMode: false,
DemoUsername: "demo",
AddUser: []string{},
} }
return CLI{NoConfigFile: false, ConfigFilePath: "bibliomane.toml", DisableStoreJWTKeyInConfig: false, ConfigFile: c} return CLI{NoConfigFile: false, ConfigFilePath: "bibliomane.toml", DisableStoreJWTKeyInConfig: false, ConfigFile: c}
} }

View File

@@ -9,3 +9,4 @@ ValidationPropertyFail = "Validation failed for '%s' property."
RegistrationDisabled = "Registration has been disabled on this instance." RegistrationDisabled = "Registration has been disabled on this instance."
UserAlreadyExists = "An user with this name already exists." UserAlreadyExists = "An user with this name already exists."
ErrorWhenCreatingUserFromStr = "Error when creating user from string %s" ErrorWhenCreatingUserFromStr = "Error when creating user from string %s"
ISBNNotFoundBabelio = "ISBN %s not found on babelio."

View File

@@ -9,3 +9,4 @@ ValidationPropertyFail = "La validation a échoué pour la propriété '%s'."
RegistrationDisabled = "La création de nouveaux comptes a été désactivée sur cette instance." RegistrationDisabled = "La création de nouveaux comptes a été désactivée sur cette instance."
UserAlreadyExists = "Un utilisateur avec le même nom existe déjà." UserAlreadyExists = "Un utilisateur avec le même nom existe déjà."
ErrorWhenCreatingUserFromStr = "Erreur lors de la création de l'utilisateur %s" ErrorWhenCreatingUserFromStr = "Erreur lors de la création de l'utilisateur %s"
ISBNNotFoundBabelio = "L'ISBN %s n'est pas sur babelio."

View File

@@ -12,6 +12,10 @@ import (
"gorm.io/gorm" "gorm.io/gorm"
) )
type TranslatedError struct {
Err error
}
type HttpError struct { type HttpError struct {
StatusCode int StatusCode int
Err error Err error
@@ -59,6 +63,10 @@ func (h HttpError) Error() string {
return fmt.Sprintf("%d: err %v", h.StatusCode, h.Err) return fmt.Sprintf("%d: err %v", h.StatusCode, h.Err)
} }
func (e TranslatedError) Error() string {
return fmt.Sprintf("%v", e.Err)
}
type apiValidationError struct { type apiValidationError struct {
Field string `json:"field"` Field string `json:"field"`
Err string `json:"error"` Err string `json:"error"`

View File

@@ -2,10 +2,14 @@ package routes
import ( import (
"errors" "errors"
"log"
"strings"
"git.artlef.fr/bibliomane/internal/appcontext" "git.artlef.fr/bibliomane/internal/appcontext"
"git.artlef.fr/bibliomane/internal/babelio"
"git.artlef.fr/bibliomane/internal/dto" "git.artlef.fr/bibliomane/internal/dto"
"git.artlef.fr/bibliomane/internal/fileutils" "git.artlef.fr/bibliomane/internal/fileutils"
"git.artlef.fr/bibliomane/internal/i18nresource"
"git.artlef.fr/bibliomane/internal/inventaire" "git.artlef.fr/bibliomane/internal/inventaire"
"git.artlef.fr/bibliomane/internal/model" "git.artlef.fr/bibliomane/internal/model"
"git.artlef.fr/bibliomane/internal/myvalidator" "git.artlef.fr/bibliomane/internal/myvalidator"
@@ -62,10 +66,40 @@ func saveInventaireBookToDb(ac appcontext.AppContext, inventaireEdition inventai
} }
book.Cover = cover book.Cover = cover
} }
if ac.Config.BookDescriptionFromBabelio {
isbn := findIsbn(&inventaireEdition)
if isbn != "" {
desc, err := babelio.GetDescriptionFromISBN(ac.Config.BabelioUrl, isbn)
if err != nil {
te, isTrError := errors.AsType[myvalidator.TranslatedError](err)
var errToPrint string
if isTrError {
errToPrint = i18nresource.GetTranslatedMessage(&ac, te.Error())
} else {
errToPrint = err.Error()
}
log.Println(errToPrint)
} else {
book.Summary = desc
}
}
}
err := ac.Db.Save(&book).Error err := ac.Db.Save(&book).Error
return &book, err return &book, err
} }
func findIsbn(inventaireEdition *inventaire.InventaireEditionDetailedSingleResult) string {
if inventaireEdition.ISBN != "" {
return strings.ReplaceAll(inventaireEdition.ISBN, "-", "")
}
if strings.HasPrefix(inventaireEdition.Id, "isbn:") {
return inventaireEdition.Id[5:]
}
return ""
}
func fetchOrCreateInventaireAuthor(ac appcontext.AppContext, inventaireAuthor *inventaire.InventaireAuthorResult) (*model.Author, error) { func fetchOrCreateInventaireAuthor(ac appcontext.AppContext, inventaireAuthor *inventaire.InventaireAuthorResult) (*model.Author, error) {
var author model.Author var author model.Author
res := ac.Db.Where("inventaire_id = ?", inventaireAuthor.ID).First(&author) res := ac.Db.Where("inventaire_id = ?", inventaireAuthor.ID).First(&author)