Skip to content
This repository was archived by the owner on Apr 1, 2025. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions semantic-analysis/semantic-analysis.cabal
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,7 @@ library
, filepath
, fused-effects ^>= 1.1
, hashable
, pathtype ^>= 0.8.1
, semantic-source ^>= 0.1.0.1
, semantic-source ^>= 0.2
, text ^>= 1.2.3.1
, transformers ^>= 0.5
, vector ^>= 0.12.3
24 changes: 11 additions & 13 deletions semantic-analysis/src/Analysis/Blob.hs
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,11 @@ module Analysis.Blob
, nullBlob
) where

import Analysis.File as A
import Analysis.Reference as A
import Data.Aeson
import Source.Language as Language
import Source.Source as Source
import qualified System.Path as Path
import qualified System.Path.PartClass as Path.PartClass
import Analysis.File as A
import Analysis.Reference as A
import Data.Aeson
import Source.Language as Language
import Source.Source as Source
Comment on lines -11 to +15
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Expect lots of this sort of thing due to autoformatter. Apologies.


-- | The source, path information, and language of a file read from disk.
data Blob = Blob
Expand All @@ -25,27 +23,27 @@ data Blob = Blob
instance FromJSON Blob where
parseJSON = withObject "Blob" $ \b -> do
src <- b .: "content"
Right pth <- fmap Path.parse (b .: "path")
pth <- b .: "path"
Comment on lines -28 to +26
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Simpler terms.

lang <- b .: "language"
let lang' = if knownLanguage lang then lang else Language.forPath pth
pure (fromSource (pth :: Path.AbsRelFile) lang' src)
pure (fromSource pth lang' src)


-- | Create a Blob from a provided path, language, and UTF-8 source.
-- The resulting Blob's span is taken from the 'totalSpan' of the source.
fromSource :: Path.PartClass.AbsRel ar => Path.File ar -> Language -> Source -> Blob
fromSource :: FilePath -> Language -> Source -> Blob
Comment on lines -36 to +34
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Simpler types.

fromSource filepath language source
= Blob source (A.File (A.Reference (Path.toAbsRel filepath) (totalSpan source)) language)
= Blob source (A.File (A.Reference filepath (totalSpan source)) language)

blobLanguage :: Blob -> Language
blobLanguage = A.fileBody . blobFile

blobPath :: Blob -> Path.AbsRelFile
blobPath :: Blob -> FilePath
blobPath = A.refPath . A.fileRef . blobFile

-- | Show FilePath for error or json outputs.
blobFilePath :: Blob -> String
blobFilePath = Path.toString . blobPath
blobFilePath = blobPath
Comment on lines -48 to +46
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I opted not to eliminate this in favour of blobPath because I didn't want to get dragged into that refactoring in the same PR. Still, it would be nice.


nullBlob :: Blob -> Bool
nullBlob = Source.null . blobSource
6 changes: 2 additions & 4 deletions semantic-analysis/src/Analysis/File.hs
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ import Data.Maybe (fromJust, listToMaybe)
import GHC.Stack
import Source.Language as Language
import Source.Span
import qualified System.Path as Path
import qualified System.Path.PartClass as Path.PartClass

-- Files

Expand All @@ -29,10 +27,10 @@ data File a = File
-- Constructors

fromBody :: HasCallStack => a -> File a
fromBody body = File (A.Reference (Path.absRel (srcLocFile srcLoc)) (spanFromSrcLoc srcLoc)) body where
fromBody body = File (A.Reference (srcLocFile srcLoc) (spanFromSrcLoc srcLoc)) body where
srcLoc = snd (fromJust (listToMaybe (getCallStack callStack)))

fromPath :: Path.PartClass.AbsRel ar => Path.File ar -> File Language
fromPath :: FilePath -> File Language
fromPath p = File (A.fromPath p) (Language.forPath p)


Expand Down
6 changes: 3 additions & 3 deletions semantic-analysis/src/Analysis/Module.hs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import Data.Foldable (foldl')
import qualified Data.Map as Map
import qualified Data.Set as Set
import qualified Data.Text as Text
import qualified System.Path as Path
import System.FilePath as Path

data Module a = Module
{ body :: Map.Map Name a -> a
Expand Down Expand Up @@ -48,9 +48,9 @@ instance Monoid (ModuleSet a) where
mempty = ModuleSet mempty

fromList :: [File (Module a)] -> ModuleSet a
fromList = ModuleSet . Map.fromList . map (\ (File ref mod) -> (refName ref , mod))
fromList = ModuleSet . Map.fromList . map (\ (File ref mod) -> (refName ref, mod))
where
refName (Reference path _) = name (Text.pack (Path.toString (Path.takeBaseName path)))
refName (Reference path _) = name (Text.pack (Path.takeBaseName path))

link :: ModuleSet a -> Module a -> Module a
link (ModuleSet ms) m = Module body' (imports m Set.\\ Map.keysSet ms) (exports m) unknown' where
Expand Down
8 changes: 4 additions & 4 deletions semantic-analysis/src/Analysis/Project.hs
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@ import Analysis.File
import Data.Text (Text)
import qualified Data.Text as T
import Source.Language
import qualified System.Path as Path
import System.FilePath (takeFileName)

-- | A 'Project' contains all the information that semantic needs
-- to execute an analysis, diffing, or graphing pass.
data Project = Project
{ projectRootDir :: Path.AbsRelDir
{ projectRootDir :: FilePath
, projectBlobs :: [Blob]
, projectLanguage :: Language
, projectExcludeDirs :: [Path.AbsRelDir]
, projectExcludeDirs :: [FilePath]
} deriving (Eq, Show)

projectName :: Project -> Text
projectName = T.pack . maybe "" Path.toString . Path.takeDirName . projectRootDir
projectName = T.pack . takeFileName . projectRootDir

projectExtensions :: Project -> [String]
projectExtensions = extensionsForLanguage . projectLanguage
Expand Down
8 changes: 3 additions & 5 deletions semantic-analysis/src/Analysis/Reference.hs
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,17 @@ module Analysis.Reference
) where

import Source.Span
import System.Path as Path
import System.Path.PartClass as Path.PartClass

-- Reference

data Reference = Reference
{ refPath :: Path.AbsRelFile
{ refPath :: FilePath
, refSpan :: Span
}
deriving (Eq, Ord, Show)


-- Constructors

fromPath :: Path.PartClass.AbsRel ar => Path.File ar -> Reference
fromPath p = Reference (Path.toAbsRel p) (point (Pos 0 0))
fromPath :: FilePath -> Reference
fromPath p = Reference p (point (Pos 0 0))
3 changes: 1 addition & 2 deletions semantic-analysis/src/Analysis/Syntax.hs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ import qualified Data.Vector as V
import qualified Source.Source as Source
import Source.Span
import System.FilePath
import qualified System.Path as Path

data Term
= Var Name
Expand Down Expand Up @@ -140,7 +139,7 @@ parseFile path = do
case (A.eitherDecodeWith A.json' (A.iparse parseGraph) contents) of
Left (_, err) -> throwError err
Right (_, Nothing) -> throwError "no root node found"
Right (_, Just root) -> pure (sourceContents, File (Reference (Path.absRel sourcePath) span) root)
Right (_, Just root) -> pure (sourceContents, File (Reference sourcePath span) root)
where
decrSpan (Span (Pos sl sc) (Pos el ec)) = Span (Pos (sl - 1) (sc - 1)) (Pos (el - 1) (ec - 1))

Expand Down
3 changes: 1 addition & 2 deletions semantic-ast/semantic-ast.cabal
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,7 @@ library
, filepath ^>= 1.4.1
, fused-effects ^>= 1.1
, Glob ^>= 0.10.0
, pathtype ^>= 0.8.1
, semantic-source ^>= 0.1.0.1
, semantic-source ^>= 0.2
, tasty ^>= 1.2.3
, tasty-hunit ^>= 0.10.0.2
, template-haskell >= 2.15 && < 2.19
Expand Down
60 changes: 28 additions & 32 deletions semantic-ast/src/AST/TestHelpers.hs
Original file line number Diff line number Diff line change
Expand Up @@ -7,29 +7,28 @@ module AST.TestHelpers
, testCorpus
) where

import Control.Applicative
import Control.Monad
import Data.Attoparsec.ByteString.Char8
import Data.Attoparsec.ByteString.Char8 as Attoparsec
import Data.ByteString (ByteString, readFile)
import Data.ByteString.Char8 (pack, unpack)
import Data.Either
import Data.Functor
import Prelude hiding (takeWhile)
import System.Exit (exitFailure)
import System.Path ((</>))
import qualified System.Path as Path
import qualified System.Path.Directory as Path
import System.FilePath.Glob
import Test.Tasty
import Test.Tasty.HUnit
import Control.Applicative
import Control.Monad
import Data.Attoparsec.ByteString.Char8
import Data.Attoparsec.ByteString.Char8 as Attoparsec
import Data.ByteString (ByteString, readFile)
import Data.ByteString.Char8 (pack, unpack)
import Data.Either
import Data.Functor
import Prelude hiding (takeWhile)
import System.Directory
import System.Exit (exitFailure)
import System.FilePath
import System.FilePath.Glob
import Test.Tasty
import Test.Tasty.HUnit

testCorpus :: (ByteString -> IO (Either String (t a))) -> Path.AbsRelFile -> IO TestTree
testCorpus :: (ByteString -> IO (Either String (t a))) -> FilePath -> IO TestTree
testCorpus parse path = do
xs <- parseCorpusFile path
case xs of
Left e -> print ("Failed to parse corpus: " <> show (Path.toString path) <> " " <> "Error: " <> show e) *> exitFailure
Right xs -> testGroup (Path.toString path) <$> traverse corpusTestCase xs
Left e -> print ("Failed to parse corpus: " <> show path <> " " <> "Error: " <> show e) *> exitFailure
Right xs -> testGroup path <$> traverse corpusTestCase xs
where
corpusTestCase (CorpusExample name code) = testCase name . either (errMsg code) pass <$> parse code
pass = const (pure ())
Expand All @@ -38,31 +37,28 @@ testCorpus parse path = do
-- Depending on whether these tests are invoked via cabal run or cabal test,
-- we might be in a project subdirectory or not, so let's make sure we're
-- in project subdirectories as needed.
findCorpus :: Path.RelDir -> IO Path.RelDir
findCorpus :: FilePath -> IO FilePath
findCorpus p = do
cwd <- Path.getCurrentDirectory
if Path.takeDirName cwd == Just (Path.relDir "haskell-tree-sitter")
cwd <- getCurrentDirectory
if takeFileName cwd == "haskell-tree-sitter"
then pure p
else pure (Path.relDir ".." </> p)
else pure (".." </> p)

-- The path is expected to be relative to the language project.
readCorpusFiles :: Path.RelDir -> IO [Path.RelFile]
readCorpusFiles :: FilePath -> IO [FilePath]
readCorpusFiles parent = do
dir <- findCorpus parent
files <- globDir1 (compile "**/*.txt") (Path.toString dir)
pure (Path.relPath <$> files)
globDir1 (compile "**/*.txt") dir

readCorpusFiles' :: Path.AbsRelDir -> IO [Path.AbsRelFile]
readCorpusFiles' dir = do
files <- globDir1 (compile "**/*.txt") (Path.toString dir)
pure (Path.file <$> files)
readCorpusFiles' :: FilePath -> IO [FilePath]
readCorpusFiles' = globDir1 (compile "**/*.txt")

data CorpusExample = CorpusExample { name :: String, code :: ByteString }
deriving (Eq, Show)

parseCorpusFile :: Path.AbsRelFile -> IO (Either String [CorpusExample])
parseCorpusFile :: FilePath -> IO (Either String [CorpusExample])
parseCorpusFile path = do
c <- Data.ByteString.readFile (Path.toString path)
c <- Data.ByteString.readFile path
pure $ parseOnly corpusParser c

corpusParser :: Parser [CorpusExample]
Expand Down
25 changes: 12 additions & 13 deletions semantic-ast/src/System/Path/Fixture.hs
Original file line number Diff line number Diff line change
Expand Up @@ -13,30 +13,29 @@ where

import Control.Concurrent
import GHC.Stack
import System.FilePath
import System.IO
import qualified System.Path as Path
import System.Path ((</>))

#if BAZEL_BUILD
import qualified Bazel.Runfiles as Bazel

type HasFixture =
( ?runfiles :: Bazel.Runfiles,
?project :: Path.RelDir,
?project :: FilePath,
HasCallStack
)

create :: IO Bazel.Runfiles
create = Bazel.create

root :: HasFixture => Path.AbsRelDir
root = Path.absRel (Bazel.rlocation ?runfiles ".")
root :: HasFixture => FilePath
root = Bazel.rlocation ?runfiles "."

absRelFile :: (HasFixture) => String -> Path.AbsRelFile
absRelFile x = Path.toAbsRel (root </> Path.relDir "semantic" </> ?project </> Path.relFile x)
absRelFile :: HasFixture => String -> FilePath
absRelFile x = root </> "semantic" </> ?project </> x

absRelDir :: HasFixture => String -> Path.AbsRelDir
absRelDir x = Path.toAbsRel (root </> Path.relDir "semantic" </> ?project </> Path.relDir x)
absRelDir :: HasFixture => String -> FilePath
absRelDir x = root </> "semantic" </> ?project </> x

#else

Expand All @@ -46,11 +45,11 @@ type HasFixture = HasCallStack
create :: IO ()
create = pure ()

absRelFile :: String -> Path.AbsRelFile
absRelFile x = Path.absRel "semantic" </> Path.relFile x
absRelFile :: String -> FilePath
absRelFile x = "semantic" </> x

absRelDir :: String -> Path.AbsRelDir
absRelDir x = Path.absRel "semantic" </> Path.relDir x
absRelDir :: String -> FilePath
absRelDir x = "semantic" </> x

#endif

Expand Down
3 changes: 1 addition & 2 deletions semantic-codeql/semantic-codeql.cabal
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ library
, fused-effects ^>= 1.1
, semantic-ast
, semantic-proto ^>= 0
, semantic-source ^>= 0.1.0.1
, semantic-source ^>= 0.2
, semantic-tags ^>= 0.0
, template-haskell >= 2.15 && < 2.19
, text ^>= 1.2.3
Expand All @@ -71,7 +71,6 @@ test-suite test
main-is: PreciseTest.hs
build-depends:
, base
, pathtype ^>= 0.8.1
, semantic-ast
, semantic-codeql
, tasty
13 changes: 8 additions & 5 deletions semantic-codeql/test/PreciseTest.hs
Original file line number Diff line number Diff line change
@@ -1,25 +1,28 @@
{-# LANGUAGE CPP, DisambiguateRecordFields, OverloadedStrings, TypeApplications, ImplicitParams #-}
{-# LANGUAGE CPP #-}
{-# LANGUAGE DisambiguateRecordFields #-}
{-# LANGUAGE ImplicitParams #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE TypeApplications #-}
{-# OPTIONS_GHC -Wno-unused-imports #-}
module Main (main) where

import AST.TestHelpers
import AST.Unmarshal
import qualified Language.CodeQL.AST as CodeQL
import Language.CodeQL.Grammar
import qualified System.Path as Path
import Test.Tasty
import qualified System.Path.Fixture as Fixture
import Test.Tasty

main :: IO ()
main = do
#if BAZEL_BUILD
rf <- Fixture.create
let ?project = Path.relDir "external/tree-sitter-ql"
let ?project = "external/tree-sitter-ql"
Comment on lines -17 to +20
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We aren't currently building with bazel, but I did my best to keep these changes bazel-friendly at least.

?runfiles = rf

let dirs = Fixture.absRelDir "test/corpus"
#else
dirs <- Path.absRel <$> CodeQL.getTestCorpusDir
dirs <- CodeQL.getTestCorpusDir
#endif
let parse = parseByteString @CodeQL.Ql @() tree_sitter_ql

Expand Down
3 changes: 1 addition & 2 deletions semantic-go/semantic-go.cabal
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ library
, fused-effects ^>= 1.1
, semantic-ast
, semantic-proto ^>= 0
, semantic-source ^>= 0.1.0.1
, semantic-source ^>= 0.2
, semantic-tags ^>= 0.0
, template-haskell >= 2.15 && < 2.19
, text ^>= 1.2.3
Expand All @@ -71,7 +71,6 @@ test-suite test
main-is: PreciseTest.hs
build-depends:
, base
, pathtype ^>= 0.8.1
, semantic-ast
, semantic-go
, tasty
Loading