Restructure into subpackages (#352)

Instead of a bunch of top-level packages, moves things like `edg_core`
into `edg.core`. Also renames some subpackages for simplicity:
`electronics_abstract_parts` -> `abstract_parts`, `electronics_lib` ->
`parts`. Should not break compatibility for those using `from edg import
*`, but will break for those importing internals.

This structure should allow this to be used as a submodule and is
overall cleaner. The compiler will detect if PolymorphicBlocks exists in
the current path, and if so, uses the qualified path.

`examples` is still independent and is not part of the public API or
package structure.

compiler/src/main/scala/edg/compiler/Compiler.scala

@@ -116,7 +116,7 @@ class AssignNamer() {
 }
 
 object Compiler {
-  final val kExpectedProtoVersion = 4
+  final val kExpectedProtoVersion = 5
 }
 
 /** Compiler for a particular design, with an associated library to elaborate references from.

compiler/src/main/scala/edg/compiler/CompilerServerMain.scala

@@ -13,8 +13,8 @@ import java.io.{File, PrintWriter, StringWriter}
 
 // a PythonInterface that uses the on-event hooks to forward stderr and stdout
 // without this, the compiler can freeze on large stdout/stderr data, possibly because of queue sizing
-class ForwardingPythonInterface(serverFile: Option[File], pythonPaths: Seq[String])
-    extends PythonInterface(serverFile, pythonPaths) {
+class ForwardingPythonInterface(pythonPaths: Seq[String] = Seq())
+    extends PythonInterface(pythonPaths = pythonPaths) {
   def forwardProcessOutput(): Unit = {
     StreamUtils.forAvailable(processOutputStream) { data =>
       System.out.print(new String(data))
@@ -94,9 +94,7 @@ object CompilerServerMain {
   }
 
   def main(args: Array[String]): Unit = {
-    val hdlServerOption = PythonInterface.serverFileOption(None) // local relative path
-    hdlServerOption.foreach { serverFile => println(s"Using local $serverFile") }
-    val pyIf = new ForwardingPythonInterface(hdlServerOption, Seq(new File(".").getAbsolutePath))
+    val pyIf = new ForwardingPythonInterface()
     (pyIf.getProtoVersion() match {
       case Errorable.Success(pyVersion) if pyVersion == Compiler.kExpectedProtoVersion => None
       case Errorable.Success(pyMismatchVersion) => Some(pyMismatchVersion.toString)

compiler/src/main/scala/edg/compiler/PythonInterface.scala

@@ -20,8 +20,8 @@ object ProtobufStdioSubprocess {
 
 class ProtobufStdioSubprocess[RequestType <: scalapb.GeneratedMessage, ResponseType <: scalapb.GeneratedMessage](
     responseType: scalapb.GeneratedMessageCompanion[ResponseType],
-    pythonPaths: Seq[String],
-    args: Seq[String]
+    pythonPaths: Seq[String] = Seq(),
+    args: Seq[String] = Seq()
 ) {
   protected val process: Either[Process, Throwable] =
     try {
@@ -128,32 +128,22 @@ class ProtobufStdioSubprocess[RequestType <: scalapb.GeneratedMessage, ResponseT
   }
 }
 
-object PythonInterface {
-  private val kHdlServerFilePath = "edg_hdl_server/__main__.py"
-  // returns the HDL server Python script if it exists locally, otherwise returns None.
-  def serverFileOption(root: Option[File] = None): Option[File] = {
-    val hdlServerFile = root match {
-      case Some(root) => new File(root, kHdlServerFilePath)
-      case None => new File(kHdlServerFilePath)
-    }
-    if (hdlServerFile.exists()) {
-      Some(hdlServerFile)
-    } else {
-      None
-    }
-  }
-}
-
 /** An interface to the Python HDL elaborator, which reads in Python HDL code and (partially) compiles them down to IR.
   * The underlying Python HDL should not change while this is open. This will not reload updated Python HDL files.
   *
-  * If the serverFile is specified, run that; otherwise use "python -m edg_hdl_server" for the global package.
+  * This invokes "python -m edg.hdl_server", using either the local or global (pip) module as available.
   */
-class PythonInterface(serverFile: Option[File], pythonPaths: Seq[String], pythonInterpreter: String = "python") {
-  val command = serverFile match { // -u for unbuffered mode
-    case Some(serverFile) => Seq(pythonInterpreter, "-u", serverFile.getAbsolutePath)
-    case None => Seq(pythonInterpreter, "-u", "-m", "edg_hdl_server")
+class PythonInterface(interpreter: String = "python", pythonPaths: Seq[String] = Seq()) {
+  val submoduleSearchPaths = if (pythonPaths.nonEmpty) pythonPaths else Seq(".")
+  val isSubmoduled = submoduleSearchPaths.map { searchPath => // check if submoduled, if so prepend the submodule name
+    new File(new File(searchPath), "PolymorphicBlocks/edg/hdl_server/__init__.py").exists()
+  }.exists(identity)
+  private val packageName = if (isSubmoduled) {
+    "PolymorphicBlocks.edg.hdl_server"
+  } else {
+    "edg.hdl_server"
   }
+  private val command = Seq(interpreter, "-u", "-m", packageName)
   protected val process = new ProtobufStdioSubprocess[edgrpc.HdlRequest, edgrpc.HdlResponse](
     edgrpc.HdlResponse,
     pythonPaths,

compiler/src/main/scala/edg/wir/LibraryConnectivityAnalysis.scala

@@ -7,11 +7,11 @@ import edgir.elem.elem
 object LibraryConnectivityAnalysis {
   // Shared library path to the base PortBridge class
   val portBridge =
-    ref.LibraryPath(target = Some(ref.LocalStep(step = ref.LocalStep.Step.Name("edg_core.PortBlocks.PortBridge"))))
+    ref.LibraryPath(target = Some(ref.LocalStep(step = ref.LocalStep.Step.Name("edg.core.PortBlocks.PortBridge"))))
   val portBridges = Set( // TODO this currently is a hack to avoid proper (multiple-level) subclass resolution
     portBridge,
     ref.LibraryPath(target =
-      Some(ref.LocalStep(step = ref.LocalStep.Step.Name("electronics_model.CircuitBlock.CircuitPortBridge")))
+      Some(ref.LocalStep(step = ref.LocalStep.Step.Name("edg.electronics_model.CircuitBlock.CircuitPortBridge")))
     )
   )
   val portBridgeOuterPort = "outer_port"

compiler/src/test/scala/edg/compiler/PythonInterfaceTest.scala

@@ -12,8 +12,8 @@ class PythonInterfaceTest extends AnyFlatSpec {
     val compiledDir = new File(getClass.getResource("").getPath)
     // above returns compiler/target/scala-2.xx/test-classes/edg/compiler, get the root repo dir
     val repoDir = compiledDir.getParentFile.getParentFile.getParentFile.getParentFile.getParentFile.getParentFile
-    val pyIf = new PythonInterface(Some(new File(repoDir, "edg_hdl_server/__main__.py")), Seq(repoDir.getAbsolutePath))
-    pyIf.indexModule("edg_core").getClass should equal(classOf[Errorable.Success[Seq[LibraryPath]]])
+    val pyIf = new PythonInterface(pythonPaths = Seq(repoDir.getAbsolutePath))
+    pyIf.indexModule("edg.core").getClass should equal(classOf[Errorable.Success[Seq[LibraryPath]]])
     pyIf.shutdown() should equal(0)
   }
 }

edg/BoardCompiler.py

@@ -3,10 +3,11 @@
 from contextlib import suppress
 from typing import Type, Optional, Tuple
 
-from edg_core import Block, ScalaCompiler, CompiledDesign
-from electronics_model import NetlistBackend, SvgPcbBackend
-from electronics_model.RefdesRefinementPass import RefdesRefinementPass
-from electronics_model.BomBackend import GenerateBom
+from .core import Block, ScalaCompiler, CompiledDesign
+from .electronics_model.NetlistBackend import NetlistBackend  # imported separately b/c mypy confuses with the modules
+from .electronics_model.SvgPcbBackend import SvgPcbBackend
+from .electronics_model.RefdesRefinementPass import RefdesRefinementPass
+from .electronics_model.BomBackend import GenerateBom
 
 
 def compile_board(design: Type[Block], target_dir_name: Optional[Tuple[str, str]]) -> CompiledDesign:
@@ -41,8 +42,8 @@ def compile_board(design: Type[Block], target_dir_name: Optional[Tuple[str, str]
       raw_file.write(compiled.design.SerializeToString())
 
   if compiled.errors:
-    import edg_core
-    raise edg_core.ScalaCompilerInterface.CompilerCheckError(f"error during compilation:\n{compiled.errors_str()}")
+    from . import core
+    raise core.ScalaCompilerInterface.CompilerCheckError(f"error during compilation:\n{compiled.errors_str()}")
 
   netlist_all = NetlistBackend().run(compiled)
   netlist_refdes = NetlistBackend().run(compiled, {'RefdesMode': 'refdes'})

edg/BoardTop.py

@@ -1,4 +1,4 @@
-from electronics_lib import *
+from .parts import *
 
 
 class BaseBoardTop(DesignTop):

edg/__init__.py

@@ -1,9 +1,9 @@
 # A metapackage for all the packages needed for electronics design with EDG
 
-from edg_core import *
-from electronics_model import *
-from electronics_abstract_parts import *
-from electronics_lib import *
+from .core import *
+from .electronics_model import *
+from .abstract_parts import *
+from .parts import *
 
 from .BoardTop import BoardTop, SimpleBoardTop, JlcBoardTop
 

electronics_abstract_parts/AbstractAnalogSwitch.py → edg/abstract_parts/AbstractAnalogSwitch.py

@@ -1,6 +1,6 @@
 from typing import List, cast, Optional, Dict
 
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import Interface
 
 

electronics_abstract_parts/AbstractAntenna.py → edg/abstract_parts/AbstractAntenna.py

@@ -1,4 +1,4 @@
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 from .PartsTable import PartsTableColumn, PartsTableRow
 from .PartsTablePart import PartsTableSelector

electronics_abstract_parts/AbstractBjt.py → edg/abstract_parts/AbstractBjt.py

@@ -1,6 +1,6 @@
 from typing import Dict
 
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 from .PartsTable import PartsTableColumn, PartsTableRow
 from .PartsTablePart import PartsTableFootprintSelector

electronics_abstract_parts/AbstractCapacitor.py → edg/abstract_parts/AbstractCapacitor.py

@@ -3,7 +3,7 @@
 from typing import Optional, cast, Dict, Any, List, Tuple, Mapping
 import math
 
-from electronics_model import *
+from ..electronics_model import *
 from .PartsTable import PartsTableColumn, PartsTableRow, PartsTable
 from .PartsTablePart import PartsTableFootprintSelector
 from .Categories import *

electronics_abstract_parts/AbstractConnector.py → edg/abstract_parts/AbstractConnector.py

@@ -1,4 +1,4 @@
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import Connector
 
 

electronics_abstract_parts/AbstractCrystal.py → edg/abstract_parts/AbstractCrystal.py

@@ -1,4 +1,4 @@
-from electronics_model import *
+from ..electronics_model import *
 from . import PartsTableFootprintSelector, PartsTableColumn, Capacitor, PartsTableRow
 from .Categories import *
 from .StandardFootprint import StandardFootprint

electronics_abstract_parts/AbstractDebugHeaders.py → edg/abstract_parts/AbstractDebugHeaders.py

@@ -1,5 +1,5 @@
 from .Categories import ProgrammingConnector
-from electronics_model import *
+from ..electronics_model import *
 
 
 @abstract_block

electronics_abstract_parts/AbstractDevices.py → edg/abstract_parts/AbstractDevices.py

@@ -1,4 +1,4 @@
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 
 

electronics_abstract_parts/AbstractDiodes.py → edg/abstract_parts/AbstractDiodes.py

@@ -1,7 +1,7 @@
 from typing import Dict
 from deprecated import deprecated
 
-from electronics_model import *
+from ..electronics_model import *
 from .DummyDevices import ForcedAnalogVoltage
 from .Categories import *
 from .PartsTable import PartsTableColumn, PartsTableRow

electronics_abstract_parts/AbstractFerriteBead.py → edg/abstract_parts/AbstractFerriteBead.py

@@ -1,6 +1,6 @@
 from typing import Optional, cast, Dict
 
-from electronics_model import *
+from ..electronics_model import *
 from .PartsTable import PartsTableColumn, PartsTableRow
 from .PartsTablePart import PartsTableFootprintSelector
 from .Categories import *

electronics_abstract_parts/AbstractFets.py → edg/abstract_parts/AbstractFets.py

@@ -1,6 +1,6 @@
 from typing import Optional, Any, Dict
 
-from electronics_model import *
+from ..electronics_model import *
 from .PartsTable import PartsTableColumn, PartsTableRow, PartsTable
 from .PartsTablePart import PartsTableFootprintSelector
 from .Categories import *

electronics_abstract_parts/AbstractFuse.py → edg/abstract_parts/AbstractFuse.py

@@ -1,6 +1,6 @@
 from typing import Optional, cast
 
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 from .PartsTable import PartsTableColumn, PartsTableRow
 from .PartsTablePart import PartsTableFootprint, PartsTableFootprintSelector

electronics_abstract_parts/AbstractInductor.py → edg/abstract_parts/AbstractInductor.py

@@ -1,6 +1,6 @@
 from typing import Dict, Optional, cast
 
-from electronics_model import *
+from ..electronics_model import *
 from .PartsTable import PartsTableColumn, PartsTableRow
 from .PartsTablePart import PartsTableFootprintSelector
 from .Categories import *

electronics_abstract_parts/AbstractJumper.py → edg/abstract_parts/AbstractJumper.py

@@ -1,4 +1,4 @@
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import DiscreteComponent, TypedJumper
 
 

electronics_abstract_parts/AbstractLed.py → edg/abstract_parts/AbstractLed.py

@@ -1,4 +1,4 @@
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 from .AbstractResistor import Resistor
 from .StandardFootprint import StandardFootprint

electronics_abstract_parts/AbstractLedDriver.py → edg/abstract_parts/AbstractLedDriver.py

@@ -1,4 +1,4 @@
-from electronics_abstract_parts import *
+from ..abstract_parts import *
 
 
 @abstract_block

electronics_abstract_parts/AbstractOpamp.py → edg/abstract_parts/AbstractOpamp.py

@@ -1,6 +1,6 @@
 from typing import Mapping, Tuple, List, NamedTuple
 
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import Analog
 
 

electronics_abstract_parts/AbstractOscillator.py → edg/abstract_parts/AbstractOscillator.py

@@ -1,6 +1,6 @@
 from abc import abstractmethod
 
-from electronics_model import *
+from ..electronics_model import *
 from . import PartsTableFootprint, PartsTableColumn, PartsTableRow, PartsTableSelector
 from .Categories import *
 

electronics_abstract_parts/AbstractPowerConverters.py → edg/abstract_parts/AbstractPowerConverters.py

@@ -1,6 +1,6 @@
 from abc import abstractmethod
 from typing import Optional
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 from .AbstractCapacitor import DecouplingCapacitor
 from .AbstractInductor import Inductor

electronics_abstract_parts/AbstractResistor.py → edg/abstract_parts/AbstractResistor.py

@@ -1,7 +1,7 @@
 import re
 from typing import Optional, cast, Mapping, Dict
 
-from electronics_model import *
+from ..electronics_model import *
 from .PartsTable import PartsTableColumn, PartsTableRow
 from .PartsTablePart import PartsTableFootprintSelector
 from .Categories import *

electronics_abstract_parts/AbstractResistorArray.py → edg/abstract_parts/AbstractResistorArray.py

@@ -1,6 +1,6 @@
 from typing import List
 
-from electronics_model import *
+from ..electronics_model import *
 from .AbstractResistor import Resistor
 from .PartsTable import PartsTableColumn, PartsTableRow
 from .PartsTablePart import PartsTableFootprintSelector

electronics_abstract_parts/AbstractSolidStateRelay.py → edg/abstract_parts/AbstractSolidStateRelay.py

@@ -1,6 +1,6 @@
 from typing import Dict
 
-from electronics_model import *
+from ..electronics_model import *
 from .MergedBlocks import MergedAnalogSource
 from .AbstractResistor import Resistor
 from .Categories import Interface

electronics_abstract_parts/AbstractSpiMemory.py → edg/abstract_parts/AbstractSpiMemory.py

@@ -1,4 +1,4 @@
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 
 

electronics_abstract_parts/AbstractSwitch.py → edg/abstract_parts/AbstractSwitch.py

@@ -1,6 +1,6 @@
 from typing import Dict
 
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 
 

electronics_abstract_parts/AbstractTestPoint.py → edg/abstract_parts/AbstractTestPoint.py

@@ -1,8 +1,8 @@
 from typing import cast
 
-from electronics_model import *
-from electronics_model.CanPort import CanLogicLink
-from electronics_model.I2cPort import I2cLink
+from ..electronics_model import *
+from ..electronics_model.CanPort import CanLogicLink
+from ..electronics_model.I2cPort import I2cLink
 from .AbstractConnector import RfConnector, RfConnectorTestPoint
 from .AbstractResistor import Resistor
 from .Categories import *

electronics_abstract_parts/AbstractTvsDiode.py → edg/abstract_parts/AbstractTvsDiode.py

@@ -1,4 +1,4 @@
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 from .AbstractDiodes import BaseDiode
 

electronics_abstract_parts/CanTransceiver.py → edg/abstract_parts/CanTransceiver.py

@@ -1,4 +1,4 @@
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 
 

electronics_abstract_parts/Categories.py → edg/abstract_parts/Categories.py

@@ -1,5 +1,5 @@
-from edg_core import BoolLike, init_in_parent
-from electronics_model import Block, abstract_block, InternalBlock
+from ..core import BoolLike, init_in_parent
+from ..electronics_model import Block, abstract_block, InternalBlock
 
 
 @abstract_block

electronics_abstract_parts/DigitalAmplifiers.py → edg/abstract_parts/DigitalAmplifiers.py

@@ -1,6 +1,6 @@
 from typing import Dict
 
-from electronics_model import *
+from ..electronics_model import *
 from .AbstractFets import SwitchFet
 from .AbstractResistor import Resistor
 from .AbstractDiodes import ZenerDiode

electronics_abstract_parts/DigitalIsolator.py → edg/abstract_parts/DigitalIsolator.py

@@ -1,4 +1,4 @@
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import Interface
 
 

electronics_abstract_parts/DummyDevices.py → edg/abstract_parts/DummyDevices.py

@@ -1,6 +1,6 @@
 from typing import Dict
 
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 
 

electronics_abstract_parts/ESeriesUtil.py → edg/abstract_parts/ESeriesUtil.py

@@ -4,7 +4,7 @@
 from collections import deque
 from typing import Sequence, Optional, TypeVar, Tuple, List, Generic, Type
 
-from electronics_model import *
+from ..electronics_model import *
 
 
 class ESeriesUtil: